/ Check-in [c4d50428]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add extension apis xRowCount, xQueryPhrase, xSetAuxdata and xGetAuxdata. And a ranking function that uses all of the above.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: c4d50428ab97f77e6721c4f8d03eaaf3ea91f3eb
User & Date: dan 2014-07-25 20:30:47
Context
2014-07-26
18:38
Add tests and fixes for bm25() function. check-in: 71d32f53 user: dan tags: fts5
2014-07-25
20:30
Add extension apis xRowCount, xQueryPhrase, xSetAuxdata and xGetAuxdata. And a ranking function that uses all of the above. check-in: c4d50428 user: dan tags: fts5
2014-07-23
19:31
Add a snippet() function to fts5. check-in: bdc58fd2 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

15
16
17
18
19
20
21

22
23
24
25
26
27
28
..
71
72
73
74
75
76
77
78

79
80
81
82
83
84
85
..
88
89
90
91
92
93
94







95
96
97
98
99
100
101
...
321
322
323
324
325
326
327



328
329
330
331
332






333
334
335
336
337
338
339
...
367
368
369
370
371
372
373










374
375
376
377
378
379
380
...
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
...
614
615
616
617
618
619
620
621




622
623
624






625
626
627
628
629
630
631
...
689
690
691
692
693
694
695
696
697
698
699
700
701

















































702
703
704
705

706
707
708
709
710
711
712
713



714














































715
716
717
718
719
720
721
...
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740

#include "fts5Int.h"

typedef struct Fts5Table Fts5Table;
typedef struct Fts5Cursor Fts5Cursor;
typedef struct Fts5Global Fts5Global;
typedef struct Fts5Auxiliary Fts5Auxiliary;


/*
** A single object of this type is allocated when the FTS5 module is 
** registered with a database handle. It is used to store pointers to
** all registered FTS5 extensions - tokenizers and auxiliary functions.
*/
struct Fts5Global {
................................................................................
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */

  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing function */

  int *aColumnSize;               /* Values for xColumnSize() */
};

/*
** Values for Fts5Cursor.csrflags
*/
#define FTS5CSR_REQUIRE_CONTENT   0x01
................................................................................

/*
** Macros to Set(), Clear() and Test() cursor flags.
*/
#define CsrFlagSet(pCsr, flag)   ((pCsr)->csrflags |= (flag))
#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
#define CsrFlagTest(pCsr, flag)  ((pCsr)->csrflags & (flag))








/*
** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
** argument is non-zero, attempt delete the shadow tables from teh database
*/
static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){
  int rc = SQLITE_OK;
................................................................................
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  Fts5Cursor **pp;



  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  sqlite3Fts5ExprFree(pCsr->pExpr);







  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_free(pCsr);
  return SQLITE_OK;
................................................................................
      CsrFlagSet(pCsr, FTS5CSR_EOF);
    }
    CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  }
  
  return rc;
}











/*
** This is the xFilter interface for the virtual table.  See
** the virtual table xFilter method documentation for additional
** information.
*/
static int fts5FilterMethod(
................................................................................
  rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
  if( rc==SQLITE_OK ){
    if( ePlan==FTS5_PLAN_MATCH ){
      char **pzErr = &pTab->base.zErrMsg;
      const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
      rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
      if( rc==SQLITE_OK ){
        rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
        if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
          CsrFlagSet(pCsr, FTS5CSR_EOF);
        }
        CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
      }
    }else{
      if( ePlan==FTS5_PLAN_ROWID ){
        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
      }
      rc = fts5NextMethod(pCursor);
    }
................................................................................
}

static int fts5ApiColumnCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
}

static int fts5ApiColumnAvgSize(Fts5Context *pCtx, int iCol, int *pnToken){




  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5StorageAvgsize(pTab->pStorage, iCol, pnToken);






}

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)
................................................................................
  i64 *piPos 
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  const u8 *a; int n;             /* Poslist for phrase iPhrase */
  n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
  return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}

static void fts5ApiCallback(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){

















































  static const Fts5ExtensionApi sApi = {
    1,                            /* iVersion */
    fts5ApiUserData,
    fts5ApiColumnCount,

    fts5ApiColumnAvgSize,
    fts5ApiTokenize,
    fts5ApiPhraseCount,
    fts5ApiPhraseSize,
    fts5ApiRowid,
    fts5ApiColumnText,
    fts5ApiColumnSize,
    fts5ApiPoslist,



  };















































  Fts5Auxiliary *pAux;
  Fts5Cursor *pCsr;
  i64 iCsrId;

  assert( argc>=1 );
  pAux = (Fts5Auxiliary*)sqlite3_user_data(context);
................................................................................
  }
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{
    assert( pCsr->pAux==0 );
    pCsr->pAux = pAux;
    pAux->xFunc(&sApi, (Fts5Context*)pCsr, context, argc-1, &argv[1]);
    pCsr->pAux = 0;
  }
}


/*
** This routine implements the xFindFunction method for the FTS3







>







 







|
>







 







>
>
>
>
>
>
>







 







>
>
>





>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>







 







|
<
<
<
<







 







|
>
>
>
>


|
>
>
>
>
>
>







 








|
|
|
|

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
|
|
|
>
|
|
|
|
|
|
|
|
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
..
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
..
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
...
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
...
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
...
427
428
429
430
431
432
433
434




435
436
437
438
439
440
441
...
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
...
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
...
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873

#include "fts5Int.h"

typedef struct Fts5Table Fts5Table;
typedef struct Fts5Cursor Fts5Cursor;
typedef struct Fts5Global Fts5Global;
typedef struct Fts5Auxiliary Fts5Auxiliary;
typedef struct Fts5Auxdata Fts5Auxdata;

/*
** A single object of this type is allocated when the FTS5 module is 
** registered with a database handle. It is used to store pointers to
** all registered FTS5 extensions - tokenizers and auxiliary functions.
*/
struct Fts5Global {
................................................................................
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */

  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */
};

/*
** Values for Fts5Cursor.csrflags
*/
#define FTS5CSR_REQUIRE_CONTENT   0x01
................................................................................

/*
** Macros to Set(), Clear() and Test() cursor flags.
*/
#define CsrFlagSet(pCsr, flag)   ((pCsr)->csrflags |= (flag))
#define CsrFlagClear(pCsr, flag) ((pCsr)->csrflags &= ~(flag))
#define CsrFlagTest(pCsr, flag)  ((pCsr)->csrflags & (flag))

struct Fts5Auxdata {
  Fts5Auxiliary *pAux;            /* Extension to which this belongs */
  void *pPtr;                     /* Pointer value */
  void(*xDelete)(void*);          /* Destructor */
  Fts5Auxdata *pNext;             /* Next object in linked list */
};

/*
** Close a virtual table handle opened by fts5InitVtab(). If the bDestroy
** argument is non-zero, attempt delete the shadow tables from teh database
*/
static int fts5FreeVtab(Fts5Table *pTab, int bDestroy){
  int rc = SQLITE_OK;
................................................................................
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  Fts5Cursor **pp;
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr->idxNum);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  sqlite3Fts5ExprFree(pCsr->pExpr);

  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

  /* Remove the cursor from the Fts5Global.pCsr list */
  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
  *pp = pCsr->pNext;

  sqlite3_free(pCsr);
  return SQLITE_OK;
................................................................................
      CsrFlagSet(pCsr, FTS5CSR_EOF);
    }
    CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  }
  
  return rc;
}

static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bAsc){
  int rc;
  rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, bAsc);
  if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
    CsrFlagSet(pCsr, FTS5CSR_EOF);
  }
  CsrFlagSet(pCsr, FTS5CSR_REQUIRE_CONTENT | FTS5CSR_REQUIRE_DOCSIZE );
  return rc;
}

/*
** This is the xFilter interface for the virtual table.  See
** the virtual table xFilter method documentation for additional
** information.
*/
static int fts5FilterMethod(
................................................................................
  rc = sqlite3Fts5StorageStmt(pTab->pStorage, eStmt, &pCsr->pStmt);
  if( rc==SQLITE_OK ){
    if( ePlan==FTS5_PLAN_MATCH ){
      char **pzErr = &pTab->base.zErrMsg;
      const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
      rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
      if( rc==SQLITE_OK ){
        rc = fts5CursorFirst(pTab, pCsr, bAsc);




      }
    }else{
      if( ePlan==FTS5_PLAN_ROWID ){
        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
      }
      rc = fts5NextMethod(pCursor);
    }
................................................................................
}

static int fts5ApiColumnCount(Fts5Context *pCtx){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  return ((Fts5Table*)(pCsr->base.pVtab))->pConfig->nCol;
}

static int fts5ApiColumnTotalSize(
  Fts5Context *pCtx, 
  int iCol, 
  sqlite3_int64 *pnToken
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken);
}

static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow);
}

static int fts5ApiTokenize(
  Fts5Context *pCtx, 
  const char *pText, int nText, 
  void *pUserData,
  int (*xToken)(void*, const char*, int, int, int, int)
................................................................................
  i64 *piPos 
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  const u8 *a; int n;             /* Poslist for phrase iPhrase */
  n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, &a);
  return sqlite3Fts5PoslistNext64(a, n, pi, piPos);
}

static int fts5ApiSetAuxdata(
  Fts5Context *pCtx,              /* Fts5 context */
  void *pPtr,                     /* Pointer to save as auxdata */
  void(*xDelete)(void*)           /* Destructor for pPtr (or NULL) */
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Auxdata *pData;

  for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
    if( pData->pAux==pCsr->pAux ) break;
  }

  if( pData ){
    if( pData->xDelete ){
      pData->xDelete(pData->pPtr);
    }
  }else{
    pData = (Fts5Auxdata*)sqlite3_malloc(sizeof(Fts5Auxdata));
    if( pData==0 ) return SQLITE_NOMEM;
    memset(pData, 0, sizeof(Fts5Auxdata));
    pData->pAux = pCsr->pAux;
    pData->pNext = pCsr->pAuxdata;
    pCsr->pAuxdata = pData;
  }

  pData->xDelete = xDelete;
  pData->pPtr = pPtr;
  return SQLITE_OK;
}

static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Auxdata *pData;
  void *pRet = 0;

  for(pData=pCsr->pAuxdata; pData; pData=pData->pNext){
    if( pData->pAux==pCsr->pAux ) break;
  }

  if( pData ){
    pRet = pData->pPtr;
    if( bClear ){
      pData->pPtr = 0;
      pData->xDelete = 0;
    }
  }

  return pRet;
}

static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
);

static const Fts5ExtensionApi sFts5Api = {
  1,                            /* iVersion */
  fts5ApiUserData,
  fts5ApiColumnCount,
  fts5ApiRowCount,
  fts5ApiColumnTotalSize,
  fts5ApiTokenize,
  fts5ApiPhraseCount,
  fts5ApiPhraseSize,
  fts5ApiRowid,
  fts5ApiColumnText,
  fts5ApiColumnSize,
  fts5ApiPoslist,
  fts5ApiQueryPhrase,
  fts5ApiSetAuxdata,
  fts5ApiGetAuxdata,
};


/*
** Implementation of API function xQueryPhrase().
*/
static int fts5ApiQueryPhrase(
  Fts5Context *pCtx, 
  int iPhrase, 
  void *pUserData,
  int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*)
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  int rc;
  Fts5Cursor *pNew = 0;

  rc = fts5OpenMethod(pCsr->base.pVtab, (sqlite3_vtab_cursor**)&pNew);
  if( rc==SQLITE_OK ){
    Fts5Config *pConf = pTab->pConfig;
    pNew->idxNum = FTS5_PLAN_MATCH;
    pNew->base.pVtab = (sqlite3_vtab*)pTab;
    rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
  }

  if( rc==SQLITE_OK ){
    for(rc = fts5CursorFirst(pTab, pNew, 0);
        rc==SQLITE_OK && CsrFlagTest(pNew, FTS5CSR_EOF)==0;
        rc = fts5NextMethod((sqlite3_vtab_cursor*)pNew)
    ){
      rc = xCallback(&sFts5Api, (Fts5Context*)pNew, pUserData);
      if( rc!=SQLITE_OK ){
        if( rc==SQLITE_DONE ) rc = SQLITE_OK;
        break;
      }
    }
  }

  fts5CloseMethod((sqlite3_vtab_cursor*)pNew);
  return rc;
}

static void fts5ApiCallback(
  sqlite3_context *context,
  int argc,
  sqlite3_value **argv
){

  Fts5Auxiliary *pAux;
  Fts5Cursor *pCsr;
  i64 iCsrId;

  assert( argc>=1 );
  pAux = (Fts5Auxiliary*)sqlite3_user_data(context);
................................................................................
  }
  if( pCsr==0 ){
    char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId);
    sqlite3_result_error(context, zErr, -1);
  }else{
    assert( pCsr->pAux==0 );
    pCsr->pAux = pAux;
    pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc-1, &argv[1]);
    pCsr->pAux = 0;
  }
}


/*
** This routine implements the xFindFunction method for the FTS3

Changes to ext/fts5/fts5.h.

39
40
41
42
43
44
45
46

47
48







49
50
51
52
53
54
55
..
69
70
71
72
73
74
75






























































76
77
78
79
80
81
82

83

84
85
86
87
88
89
90
91
92
93
94
95
96






97
98
99
100
101
102
103
104
105
106
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
);

/*
**
** xUserData:

**   Return a copy of the context pointer the extension function was 
**   registered with.







**
** xColumnCount:
**   Returns the number of columns in the FTS5 table.
**
** xColumnSize:
**   Reports the size in tokens of a column value from the current row.
**
................................................................................
** xPoslist:
**   Iterate through instances of phrase iPhrase in the current row. 
**
**   At EOF, a non-zero value is returned and output variable iPos set to -1.
**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.






























































*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 1 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);

  int (*xColumnAvgSize)(Fts5Context*, int iCol, int *pnToken);

  int (*xTokenize)(Fts5Context*, 
    const char *pText, int nText, /* Text to tokenize */
    void *pCtx,                   /* Context passed to xToken() */
    int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
  );

  int (*xPhraseCount)(Fts5Context*);
  int (*xPhraseSize)(Fts5Context*, int iPhrase);

  sqlite3_int64 (*xRowid)(Fts5Context*);
  int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
  int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
  int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);






};

#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)

/* 
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
#endif /* _FTS5_H */








|
>


>
>
>
>
>
>
>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







>
|
>













>
>
>
>
>
>










39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
..
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
);

/*
**
** xUserData(pFts):
**
**   Return a copy of the context pointer the extension function was 
**   registered with.
**
**
** xColumnTotalSize(pFts, iCol, pnToken):
**
**   Returns the total number of tokens in column iCol, considering all
**   rows in the FTS5 table.
**
**
** xColumnCount:
**   Returns the number of columns in the FTS5 table.
**
** xColumnSize:
**   Reports the size in tokens of a column value from the current row.
**
................................................................................
** xPoslist:
**   Iterate through instances of phrase iPhrase in the current row. 
**
**   At EOF, a non-zero value is returned and output variable iPos set to -1.
**
** xTokenize:
**   Tokenize text using the tokenizer belonging to the FTS5 table.
**
**
** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback):
**
**   This API function is used to query the FTS table for phrase iPhrase
**   of the current query. Specifically, a query equivalent to:
**
**       ... FROM ftstable WHERE ftstable MATCH $p ORDER BY DESC
**
**   with $p set to a phrase equivalent to the phrase iPhrase of the
**   current query is executed. For each row visited, the callback function
**   passed as the fourth argument is invoked. The context and API objects 
**   passed to the callback function may be used to access the properties of
**   each matched row. Invoking Api.xUserData() returns a copy of the pointer
**   passed as the third argument to pUserData.
**
**   If the callback function returns any value other than SQLITE_OK, the
**   query is abandoned and the xQueryPhrase function returns immediately.
**   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
**   Otherwise, the error code is propagated upwards.
**
**   If the query runs to completion without incident, SQLITE_OK is returned.
**   Or, if some error occurs before the query completes or is aborted by
**   the callback, an SQLite error code is returned.
**
**
** xSetAuxdata(pFts5, pAux, xDelete)
**
**   Save the pointer passed as the second argument as the extension functions 
**   "auxiliary data". The pointer may then be retrieved by the current or any
**   future invocation of the same fts5 extension function made as part of
**   of the same MATCH query using the xGetAuxdata() API.
**
**   Each extension function is allocated a single auxiliary data slot per
**   query. If the extension function is invoked more than once by the SQL
**   query, then all invocations share a single auxiliary data context.
**
**   If there is already an auxiliary data pointer when this function is
**   invoked, then it is replaced by the new pointer. If an xDelete callback
**   was specified along with the original pointer, it is invoked at this
**   point.
**
**   The xDelete callback, if one is specified, is also invoked on the
**   auxiliary data pointer after the FTS5 query has finished.
**
**
** xGetAuxdata(pFts5, bClear)
**
**   Returns the current auxiliary data pointer for the fts5 extension 
**   function. See the xSetAuxdata() method for details.
**
**   If the bClear argument is non-zero, then the auxiliary data is cleared
**   (set to NULL) before this function returns. In this case the xDelete,
**   if any, is not invoked.
**
**
** xRowCount(pFts5, pnRow)
**
**   This function is used to retrieve the total number of rows in the table.
**   In other words, the same value that would be returned by:
**
**        SELECT count(*) FROM ftstable;
*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 1 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);
  int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
  int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);

  int (*xTokenize)(Fts5Context*, 
    const char *pText, int nText, /* Text to tokenize */
    void *pCtx,                   /* Context passed to xToken() */
    int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
  );

  int (*xPhraseCount)(Fts5Context*);
  int (*xPhraseSize)(Fts5Context*, int iPhrase);

  sqlite3_int64 (*xRowid)(Fts5Context*);
  int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn);
  int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken);
  int (*xPoslist)(Fts5Context*, int iPhrase, int *pi, sqlite3_int64 *piPos);

  int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
    int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
  );
  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
  void *(*xGetAuxdata)(Fts5Context*, int bClear);
};

#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF)

/* 
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/
#endif /* _FTS5_H */

Changes to ext/fts5/fts5Int.h.

296
297
298
299
300
301
302
303

304
305
306
307
308
309
310
...
349
350
351
352
353
354
355


356
357
358
359
360
361
362

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg);



/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


................................................................................

/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(sqlite3*);

int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);



/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y.  */

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);







|
>







 







>
>







296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
...
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);


/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


................................................................................

/* Called during startup to register a UDF with SQLite */
int sqlite3Fts5ExprInit(sqlite3*);

int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);

int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**);

/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y.  */

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);

Changes to ext/fts5/fts5_aux.c.

8
9
10
11
12
13
14

15
16
17
18
19
20
21
...
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
...
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
...
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
...
409
410
411
412
413
414
415
416























































































417
418
419
420
421
422
423
424





























































































425
426
427
428
429
430
431
...
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
...
576
577
578
579
580
581
582
583





































584
585
586
587
588
589
590





591




592
593
594
595
596
597
598
...
625
626
627
628
629
630
631

632
633
634
635
636
637
638
639
640
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#include "fts5Int.h"


typedef struct SnippetPhrase SnippetPhrase;
typedef struct SnippetIter SnippetIter;
typedef struct SnippetCtx SnippetCtx;

struct SnippetPhrase {
  u64 mask;                       /* Current mask */
................................................................................
  rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback);
  if( rc==SQLITE_OK ){
    int i1;                       /* First token from input to include */
    int i2;                       /* Last token from input to include */

    int iPrint;
    int iMatchto;
    int iBit0;
    int iLast;

    int *aiStart = ctx.aiStart - ctx.iFirst;
    int *aiEnd = ctx.aiEnd - ctx.iFirst;

    /* Ideally we want to start the snippet with token (ctx.iFirst + nShift).
    ** However, this is only possible if there are sufficient tokens within
................................................................................
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  const char *zStart = "<b>";
  const char *zFinal = "</b>";
  const char *zEllip = "<b>...</b>";
  int nToken = -15;
  int nAbs;
  int nFrag;                      /* Number of fragments to return */
  int rc;
  SnippetIter *pIter = 0;

  if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
  if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
  if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
  if( nVal>=4 ){
................................................................................
  }
  nAbs = nToken * (nToken<0 ? -1 : 1);

  rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
  if( rc==SQLITE_OK ){
    Fts5Buffer buf;               /* Result buffer */
    int nBestScore = 0;           /* Score of best snippet found */
    int n;                        /* Size of column snippet is from in bytes */
    int i;                        /* Used to iterate through phrases */

    for(fts5SnippetIterFirst(pIter); 
        pIter->iLast>=0; 
        fts5SnippetIterNext(pIter)
    ){
      if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
    }
................................................................................
  }

  fts5SnippetIterFree(pIter);
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}
























































































static void fts5Bm25Function(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  assert( 0 );





























































































}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
................................................................................
    zReq = (const char*)sqlite3_value_text(apVal[0]);
  }

  memset(&s, 0, sizeof(Fts5Buffer));
  nCol = pApi->xColumnCount(pFts);

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columnavgsize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnavgsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      int colsz = 0;
      rc = pApi->xColumnAvgSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
................................................................................
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
    }
    sqlite3_free(s3.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " rowid ");





































  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){





    sqlite3Fts5BufferAppendPrintf(&rc, &s, " tokenize ");




  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
................................................................................
int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {

    { "snippet", 0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction, 0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
    rc = sqlite3Fts5CreateAux(pGlobal, 







>







 







<







 







<







 







<
<







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|

|


|
|







 







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







>
>
>
>
>
|
>
>
>
>







 







>
|
|







8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
...
264
265
266
267
268
269
270

271
272
273
274
275
276
277
...
363
364
365
366
367
368
369

370
371
372
373
374
375
376
...
379
380
381
382
383
384
385


386
387
388
389
390
391
392
...
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507

508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
...
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
...
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
...
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
******************************************************************************
*/

#include "fts5Int.h"
#include <math.h>

typedef struct SnippetPhrase SnippetPhrase;
typedef struct SnippetIter SnippetIter;
typedef struct SnippetCtx SnippetCtx;

struct SnippetPhrase {
  u64 mask;                       /* Current mask */
................................................................................
  rc = pApi->xTokenize(pFts, zCol, nCol, (void*)&ctx, fts5SnippetCallback);
  if( rc==SQLITE_OK ){
    int i1;                       /* First token from input to include */
    int i2;                       /* Last token from input to include */

    int iPrint;
    int iMatchto;

    int iLast;

    int *aiStart = ctx.aiStart - ctx.iFirst;
    int *aiEnd = ctx.aiEnd - ctx.iFirst;

    /* Ideally we want to start the snippet with token (ctx.iFirst + nShift).
    ** However, this is only possible if there are sufficient tokens within
................................................................................
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  const char *zStart = "<b>";
  const char *zFinal = "</b>";
  const char *zEllip = "<b>...</b>";
  int nToken = -15;
  int nAbs;

  int rc;
  SnippetIter *pIter = 0;

  if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
  if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
  if( nVal>=3 ) zEllip = (const char*)sqlite3_value_text(apVal[2]);
  if( nVal>=4 ){
................................................................................
  }
  nAbs = nToken * (nToken<0 ? -1 : 1);

  rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
  if( rc==SQLITE_OK ){
    Fts5Buffer buf;               /* Result buffer */
    int nBestScore = 0;           /* Score of best snippet found */



    for(fts5SnippetIterFirst(pIter); 
        pIter->iLast>=0; 
        fts5SnippetIterNext(pIter)
    ){
      if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
    }
................................................................................
  }

  fts5SnippetIterFree(pIter);
  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}

typedef struct Fts5GatherCtx Fts5GatherCtx;
struct Fts5GatherCtx {
  int nCol;
  int iPhrase;
  int *anVal;
};

static int fts5GatherCallback(
  const Fts5ExtensionApi *pApi, 
  Fts5Context *pFts,
  void *pUserData
){
  Fts5GatherCtx *p = (Fts5GatherCtx*)pUserData;
  int i = 0;
  int iPrev = -1;
  i64 iPos = 0;

  while( 0==pApi->xPoslist(pFts, 0, &i, &iPos) ){
    int iCol = FTS5_POS2COLUMN(iPos);
    if( iCol!=iPrev ){
      p->anVal[p->iPhrase * p->nCol + iCol]++;
      iPrev = iCol;
    }
  }

  return SQLITE_OK;
}

/*
** This function returns a pointer to an array of integers containing entries
** indicating the number of rows in the table for which each phrase features 
** at least once in each column.
**
** If nCol is the number of matchable columns in the table, and nPhrase is
** the number of phrases in the query, the array contains a total of
** (nPhrase*nCol) entries.
**
** For phrase iPhrase and column iCol:
**
**   anVal[iPhrase * nCol + iCol]
**
** is set to the number of rows in the table for which column iCol contains 
** at least one instance of phrase iPhrase.
*/
static int fts5GatherTotals(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  int **panVal
){
  int rc = SQLITE_OK;
  int *anVal = 0;
  int i;                          /* For iterating through expression phrases */
  int nPhrase = pApi->xPhraseCount(pFts);
  int nCol = pApi->xColumnCount(pFts);
  int nByte = nCol * nPhrase * sizeof(int);
  Fts5GatherCtx sCtx;

  sCtx.nCol = nCol;
  anVal = sCtx.anVal = (int*)sqlite3_malloc(nByte);
  if( anVal==0 ){
    rc = SQLITE_NOMEM;
  }else{
    memset(anVal, 0, nByte);
  }

  for(i=0; i<nPhrase && rc==SQLITE_OK; i++){
    sCtx.iPhrase = i;
    rc = pApi->xQueryPhrase(pFts, i, (void*)&sCtx, fts5GatherCallback);
  }

  if( rc!=SQLITE_OK ){
    sqlite3_free(anVal);
    anVal = 0;
  }

  *panVal = anVal;
  return rc;
}

typedef struct Fts5Bm25Context Fts5Bm25Context;
struct Fts5Bm25Context {
  int nPhrase;
  int nCol;
  double *aIDF;                   /* Array of IDF values */
  double *aAvg;                   /* Average size of each column in tokens */
};

static void fts5Bm25Function(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){

  const double k1 = 1.2;
  const double B = 0.75;

  int rc = SQLITE_OK;
  Fts5Bm25Context *p;

  p = pApi->xGetAuxdata(pFts, 0);
  if( p==0 ){
    int *anVal = 0;
    int ic;                       /* For iterating through columns */
    int ip;                       /* For iterating through phrases */
    i64 nRow;                     /* Total number of rows in table */
    int nPhrase = pApi->xPhraseCount(pFts);
    int nCol = pApi->xColumnCount(pFts);
    int nByte = sizeof(Fts5Bm25Context) 
              + sizeof(double) * nPhrase * nCol       /* aIDF[] */
              + sizeof(double) * nCol;                /* aAvg[] */

    p = (Fts5Bm25Context*)sqlite3_malloc(nByte);
    if( p==0 ){
      rc = SQLITE_NOMEM;
    }else{
      memset(p, 0, nByte);
      p->aAvg = (double*)&p[1];
      p->aIDF = (double*)&p->aAvg[nCol];
    }

    if( rc==SQLITE_OK ){
      rc = pApi->xRowCount(pFts, &nRow); 
      assert( nRow>0 || rc!=SQLITE_OK );
    }

    for(ic=0; rc==SQLITE_OK && ic<nCol; ic++){
      i64 nToken = 0;
      rc = pApi->xColumnTotalSize(pFts, ic, &nToken);
      p->aAvg[ic] = (double)nToken / (double)nRow;
    }

    if( rc==SQLITE_OK ){
      rc = fts5GatherTotals(pApi, pFts, &anVal);
    }
    for(ic=0; ic<nCol; ic++){
      for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
        int idx = ip * nCol + ic;
        p->aIDF[idx] = log( (0.5 + nRow - anVal[idx]) / (0.5 + anVal[idx]) );
        if( p->aIDF[idx]<0.0 ) p->aIDF[idx] = 0.0;
      }
    }

    sqlite3_free(anVal);
    if( rc==SQLITE_OK ){
      rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
    }
    if( rc!=SQLITE_OK ){
      sqlite3_free(p);
    }
  }

  if( rc==SQLITE_OK ){
    int ip;
    double score = 0.0;

    for(ip=0; rc==SQLITE_OK && ip<p->nPhrase; ip++){
      int iPrev = 0;
      int nHit = 0;
      int i = 0;
      i64 iPos = 0;

      while( rc==SQLITE_OK && 0==pApi->xPoslist(pFts, ip, &i, &iPos) ){
        int iCol = FTS5_POS2COLUMN(iPos);
        if( iCol!=iPrev && nHit>0 ){
          int sz = 0;
          int idx = ip * p->nCol + iPrev;
          rc = pApi->xColumnSize(pFts, iPrev, &sz);

          score += p->aIDF[idx] * nHit * (k1+1.0) / 
                  (nHit + k1 * (1.0 - B + B * sz / p->aAvg[iCol]));
          nHit = 0;
        }
        nHit++;
        iPrev = iCol;
      }
    }
    
    if( rc==SQLITE_OK ){
      sqlite3_result_double(pCtx, score);
    }

  }

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
................................................................................
    zReq = (const char*)sqlite3_value_text(apVal[0]);
  }

  memset(&s, 0, sizeof(Fts5Buffer));
  nCol = pApi->xColumnCount(pFts);

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      i64 colsz = 0;
      rc = pApi->xColumnTotalSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "columncount ");
................................................................................
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
    }
    sqlite3_free(s3.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){
    int ic, ip;
    int *anVal = 0;
    Fts5Buffer buf1;
    memset(&buf1, 0, sizeof(Fts5Buffer));

    if( rc==SQLITE_OK ){
      anVal = (int*)pApi->xGetAuxdata(pFts, 0);
      if( anVal==0 ){
        rc = fts5GatherTotals(pApi, pFts, &anVal);
        if( rc==SQLITE_OK ){
          rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free);
        }
      }
    }

    for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
      if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " ");
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{");
      for(ic=0; ic<nCol; ic++){
        int iVal = anVal[ip * nCol + ic];
        sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal);
      }
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}");
    }

    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p);
    }
    sqlite3_free(buf1.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowcount ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){
    i64 nRow;
    rc = pApi->xRowCount(pFts, &nRow);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " tokenize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
................................................................................
int sqlite3Fts5AuxInit(Fts5Global *pGlobal){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
    { "bm25",      0, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction,    0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  for(i=0; rc==SQLITE_OK && i<sizeof(aBuiltin)/sizeof(aBuiltin[0]); i++){
    rc = sqlite3Fts5CreateAux(pGlobal, 

Changes to ext/fts5/fts5_expr.c.

222
223
224
225
226
227
228




































































































229
230
231
232
233
234
235
    }
  }

  sqlite3_free(sParse.apPhrase);
  *pzErr = sParse.zErr;
  return sParse.rc;
}





































































































/*
** Free the expression node object passed as the only argument.
*/
void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pLeft);







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    }
  }

  sqlite3_free(sParse.apPhrase);
  *pzErr = sParse.zErr;
  return sParse.rc;
}

static char *fts5ExprStrdup(int *pRc, const char *zIn){
  char *zRet = 0;
  if( *pRc==SQLITE_OK ){
    int nByte = strlen(zIn) + 1;
    zRet = sqlite3_malloc(nByte);
    if( zRet ){
      memcpy(zRet, zIn, nByte);
    }else{
      *pRc = SQLITE_NOMEM;
    }
  }
  return zRet;
}

static void *fts5ExprMalloc(int *pRc, int nByte){
  void *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_malloc(nByte);
    if( pRet ){
      memset(pRet, 0, nByte);
    }else{
      *pRc = SQLITE_NOMEM;
    }
  }
  return pRet;
}

/*
** Create a new FTS5 expression by cloning phrase iPhrase of the
** expression passed as the second argument.
*/
int sqlite3Fts5ExprPhraseExpr(
  Fts5Config *pConfig,
  Fts5Expr *pExpr, 
  int iPhrase, 
  Fts5Expr **ppNew
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
  int i;                          /* Used to iterate through phrase terms */

  /* Components of the new expression object */
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;

  pOrig = pExpr->apPhrase[iPhrase];
  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );

  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){
    /* All the allocations succeeded. Put the expression object together. */
    pNew->pIndex = pExpr->pIndex;
    pNew->pRoot = pNode;
    pNew->nPhrase = 1;
    pNew->apPhrase = apPhrase;
    pNew->apPhrase[0] = pCopy;

    pNode->eType = FTS5_STRING;
    pNode->pNear = pNear;

    pNear->iCol = -1;
    pNear->nPhrase = 1;
    pNear->apPhrase[0] = pCopy;

    pCopy->nTerm = pOrig->nTerm;
    pCopy->pNode = pNode;
  }else{
    /* At least one allocation failed. Free them all. */
    if( pCopy ){
      for(i=0; i<pOrig->nTerm; i++){
        sqlite3_free(pCopy->aTerm[i].zTerm);
      }
      sqlite3_free(pCopy);
      sqlite3_free(pNear);
      sqlite3_free(pNode);
      sqlite3_free(apPhrase);
      sqlite3_free(pNew);
      pNew = 0;
    }
  }

  *ppNew = pNew;
  return rc;
}

/*
** Free the expression node object passed as the only argument.
*/
void sqlite3Fts5ParseNodeFree(Fts5ExprNode *p){
  if( p ){
    sqlite3Fts5ParseNodeFree(p->pLeft);

Changes to ext/fts5/fts5_storage.c.

701
702
703
704
705
706
707
708
709
710
711
712

713
714
715
716





717
718
719
720
    if( bCorrupt && rc==SQLITE_OK ){
      rc = SQLITE_CORRUPT_VTAB;
    }
  }
  return rc;
}

int sqlite3Fts5StorageAvgsize(Fts5Storage *p, int iCol, int *pnAvg){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    int nAvg = 1;
    if( p->nTotalRow ){

      nAvg = (int)((p->aTotalSize[iCol] + (p->nTotalRow/2)) / p->nTotalRow);
      if( nAvg<1 ) nAvg = 1;
      *pnAvg = nAvg;
    }





  }
  return rc;
}








|


|
<
>
|
<
<
|
>
>
>
>
>




701
702
703
704
705
706
707
708
709
710
711

712
713


714
715
716
717
718
719
720
721
722
723
    if( bCorrupt && rc==SQLITE_OK ){
      rc = SQLITE_CORRUPT_VTAB;
    }
  }
  return rc;
}

int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    *pnToken = p->aTotalSize[iCol];

  }
  return rc;


}

int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){
  int rc = fts5StorageLoadTotals(p);
  if( rc==SQLITE_OK ){
    *pnRow = p->nTotalRow;
  }
  return rc;
}

Changes to test/fts5ae.test.

153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
...
187
188
189
190
191
192
193
194







































195
196
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {2 2}
  2 {2 2}
  1 {2 2}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test(t5, 'columnavgsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {2 3}
  2 {2 3}
  1 {2 3}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
do_execsql_test 6.1 {
................................................................................

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}








































finish_test








|


|
|
|




|


|
|
|







 








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
...
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {5 7}
  2 {5 7}
  1 {5 7}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {8 12}
  2 {8 12}
  1 {8 12}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
do_execsql_test 6.1 {
................................................................................

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
}
do_test 7.2 {
  foreach {x y} {
    {q i b w s a a e l o} {i b z a l f p t e u}
    {b a z t a l o x d i} {b p a d b f h d w y}
    {z m h n p p u i e g} {v h d v b x j j c z}
    {a g i m v a u c b i} {p k s o t l r t b m}
    {v v c j o d a s c p} {f f v o k p o f o g}
  } {
    execsql {INSERT INTO t7 VALUES($x, $y)}
  }
  execsql { SELECT count(*) FROM t7 }
} {5}

foreach {tn q res} {
  1 a {{4 2}}
  2 b {{3 4}}
  3 c {{2 1}}
  4 d {{2 2}}
  5 {a AND b} {{4 2} {3 4}}
  6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
  do_execsql_test 7.3.$tn { 
    SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1
  } [list $res]
}

do_execsql_test 7.4 {
  SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}



finish_test

Changes to test/fts5af.test.

26
27
28
29
30
31
32



















33
34
35
36
37
38
39
..
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
...
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
}


do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
}





















foreach {tn doc res} {

  1.1 {X o o o o o o} {[X] o o o o o o}
  1.2 {o X o o o o o} {o [X] o o o o o}
  1.3 {o o X o o o o} {o o [X] o o o o}
  1.4 {o o o X o o o} {o o o [X] o o o}
................................................................................
  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
} {
  do_execsql_test 1.$tn.1 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES($doc, NULL);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
  } [list $res]

  do_execsql_test 1.$tn.2 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(NULL, $doc);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X';
  } [list $res]
}

foreach {tn doc res} {
  1.1 {X Y o o o o o} {[X Y] o o o o o}
  1.2 {o X Y o o o o} {o [X Y] o o o o}
  1.3 {o o X Y o o o} {o o [X Y] o o o}
  1.4 {o o o X Y o o} {o o o [X Y] o o}
................................................................................
  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}

} {
  do_execsql_test 2.$tn.1 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES($doc, NULL);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
  } [list $res]

  do_execsql_test 2.$tn.2 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(NULL, $doc);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH 'X+Y';
  } [list $res]
}

finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
<
<
<
<
<
<
<
<
<
<







 







|
<
<
<
<
<
<
<
<
<
<




26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
..
97
98
99
100
101
102
103
104










105
106
107
108
109
110
111
...
126
127
128
129
130
131
132
133










134
135
136
137
}


do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
}

proc do_snippet_test {tn doc match res} {

  uplevel #0 [list set v1 $doc]
  uplevel #0 [list set v2 $match]

  do_execsql_test $tn.1 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES($v1, NULL);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
  } [list $res]

  do_execsql_test $tn.2 {
    DELETE FROM t1;
    INSERT INTO t1 VALUES(NULL, $v1);
    SELECT snippet(t1, '[', ']', '...', 7) FROM t1 WHERE t1 MATCH $v2;
  } [list $res]

}


foreach {tn doc res} {

  1.1 {X o o o o o o} {[X] o o o o o o}
  1.2 {o X o o o o o} {o [X] o o o o o}
  1.3 {o o X o o o o} {o o [X] o o o o}
  1.4 {o o o X o o o} {o o o [X] o o o}
................................................................................
  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
} {
  do_snippet_test 1.$tn $doc X $res










}

foreach {tn doc res} {
  1.1 {X Y o o o o o} {[X Y] o o o o o}
  1.2 {o X Y o o o o} {o [X Y] o o o o}
  1.3 {o o X Y o o o} {o o [X Y] o o o}
  1.4 {o o o X Y o o} {o o o [X Y] o o}
................................................................................
  3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
  3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
  3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
  3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
  3.8 {o o o o o o o X Y} {...o o o o o [X Y]}

} {
  do_snippet_test 1.$tn $doc "X + Y" $res










}

finish_test