/ Check-in [8c30605b]
Login
Home Timeline Branches Tags Tickets Wiki More...

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: 8c30605bcd0a78a5015948171145bc6f640b8358
User & Date: dan 2015-12-29 19:35:03
Context
2015-12-30
19:58
Updates to fts5 to support detail=none mode. As of this commit, many cases are still broken. (check-in: ac8f4cf0 user: dan tags: fts5-offsets)
2015-12-29
19:35
Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase. (check-in: 8c30605b user: dan tags: fts5-offsets)
2015-12-28
19:55
Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. (check-in: 228b4d10 user: dan tags: fts5-offsets)
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.h. 

212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
...
242
243
244
245
246
247
248
249
250



251
252
253
254
255
256
257

**   modify this structure directly - it should only be used as shown above
**   with the xPhraseFirst() and xPhraseNext() API methods.
**
** xPhraseNext()
**   See xPhraseFirst above.
*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 2 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);
  int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
  int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);

................................................................................

  int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
    int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
  );
  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
  void *(*xGetAuxdata)(Fts5Context*, int bClear);

  void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
  void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);



};

/* 
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/

/*************************************************************************








|







 







|

>
>
>








212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
...
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260

**   modify this structure directly - it should only be used as shown above
**   with the xPhraseFirst() and xPhraseNext() API methods.
**
** xPhraseNext()
**   See xPhraseFirst above.
*/
struct Fts5ExtensionApi {
  int iVersion;                   /* Currently always set to 3 */

  void *(*xUserData)(Fts5Context*);

  int (*xColumnCount)(Fts5Context*);
  int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow);
  int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken);

................................................................................

  int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData,
    int(*)(const Fts5ExtensionApi*,Fts5Context*,void*)
  );
  int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
  void *(*xGetAuxdata)(Fts5Context*, int bClear);

  int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
  void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);

  int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
  void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
};

/* 
** CUSTOM AUXILIARY FUNCTIONS
*************************************************************************/

/*************************************************************************

Changes to ext/fts5/fts5Int.h. 

443
444
445
446
447
448
449


450
451
452
453
454
455
456
...
642
643
644
645
646
647
648


649
650
651
652
653
654
655


int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);

int sqlite3Fts5IndexLoadConfig(Fts5Index *p);



/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_varint.c. 
*/
................................................................................

Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
);

int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);



/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y.  */

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);








>
>







 







>
>








443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
...
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659


int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);

int sqlite3Fts5IndexLoadConfig(Fts5Index *p);

int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);

/*
** End of interface to code in fts5_index.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_varint.c. 
*/
................................................................................

Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
);

int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);

int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by
** the parser code in fts5parse.y.  */

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);

Changes to ext/fts5/fts5_expr.c. 

2322
2323
2324
2325
2326
2327
2328
2329























  sCtx.aWriter = aWriter;
  sCtx.iOff = (((i64)iCol) << 32) - 1;

  return sqlite3Fts5Tokenize(pConfig, 
      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
  );
}
































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351

  sCtx.aWriter = aWriter;
  sCtx.iOff = (((i64)iCol) << 32) - 1;

  return sqlite3Fts5Tokenize(pConfig, 
      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
  );
}

/*
** This function is only called for detail=columns tables. 
*/
int sqlite3Fts5ExprPhraseCollist(
  Fts5Expr *pExpr, 
  int iPhrase, 
  const u8 **ppCollist, 
  int *pnCollist
){
  Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
  Fts5ExprNode *pNode = pPhrase->pNode;
  assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );

  if( pNode->bEof==0 && pNode->iRowid==pExpr->pRoot->iRowid ){
    sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
  }else{
    *ppCollist = 0;
    *pnCollist = 0;
  }
  return SQLITE_OK;
}

Changes to ext/fts5/fts5_index.c. 

4876
4877
4878
4879
4880
4881
4882











4883
4884
4885
4886
4887
4888
4889

    if( eDetail==FTS5_DETAIL_FULL ){
      *pp = pIter->poslist.p;
    }
    *pn = pIter->poslist.n;
  }
  return fts5IndexReturn(pIter->pIndex);
}












/*
** This function is similar to sqlite3Fts5IterPoslist(), except that it
** copies the position list into the buffer supplied as the second 
** argument.
*/
int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){








>
>
>
>
>
>
>
>
>
>
>








4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900

    if( eDetail==FTS5_DETAIL_FULL ){
      *pp = pIter->poslist.p;
    }
    *pn = pIter->poslist.n;
  }
  return fts5IndexReturn(pIter->pIndex);
}

int sqlite3Fts5IterCollist(
  Fts5IndexIter *pIter, 
  const u8 **pp,                  /* OUT: Pointer to position-list data */
  int *pn                         /* OUT: Size of position-list in bytes */
){
  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  *pp = pIter->poslist.p;
  *pn = pIter->poslist.n;
  return SQLITE_OK;
}

/*
** This function is similar to sqlite3Fts5IterPoslist(), except that it
** copies the position list into the buffer supplied as the second 
** argument.
*/
int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){

Changes to ext/fts5/fts5_main.c. 

1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
































































1991
1992
1993
1994
1995
1996
1997
....
2008
2009
2010
2011
2012
2013
2014


2015
2016
2017
2018
2019
2020
2021

  Fts5Context *pCtx, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
){
  if( pIter->a>=pIter->b ){
    *piCol = -1;
    *piOff = -1;
#if 0
  }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    *piCol += (iVal-2);
    *piOff = -1;
#endif
  }else{
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    if( iVal==1 ){
      pIter->a += fts5GetVarint32(pIter->a, iVal);
      *piCol = iVal;
      *piOff = 0;
      pIter->a += fts5GetVarint32(pIter->a, iVal);
    }
    *piOff += (iVal-2);
  }
}

static void fts5ApiPhraseFirst(
  Fts5Context *pCtx, 
  int iPhrase, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
  pIter->b = &pIter->a[n];
  *piCol = 0;
  *piOff = 0;
  fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
}

































































static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
);

static const Fts5ExtensionApi sFts5Api = {
  2,                            /* iVersion */
................................................................................
  fts5ApiColumnText,
  fts5ApiColumnSize,
  fts5ApiQueryPhrase,
  fts5ApiSetAuxdata,
  fts5ApiGetAuxdata,
  fts5ApiPhraseFirst,
  fts5ApiPhraseNext,


};

/*
** Implementation of API function xQueryPhrase().
*/
static int fts5ApiQueryPhrase(
  Fts5Context *pCtx, 








<
<
<
<
<
<
<













|












>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>
>








1951
1952
1953
1954
1955
1956
1957







1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
....
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080

  Fts5Context *pCtx, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
){
  if( pIter->a>=pIter->b ){
    *piCol = -1;
    *piOff = -1;







  }else{
    int iVal;
    pIter->a += fts5GetVarint32(pIter->a, iVal);
    if( iVal==1 ){
      pIter->a += fts5GetVarint32(pIter->a, iVal);
      *piCol = iVal;
      *piOff = 0;
      pIter->a += fts5GetVarint32(pIter->a, iVal);
    }
    *piOff += (iVal-2);
  }
}

static int fts5ApiPhraseFirst(
  Fts5Context *pCtx, 
  int iPhrase, 
  Fts5PhraseIter *pIter, 
  int *piCol, int *piOff
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
  pIter->b = &pIter->a[n];
  *piCol = 0;
  *piOff = 0;
  fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
}

static void fts5ApiPhraseNextColumn(
  Fts5Context *pCtx, 
  Fts5PhraseIter *pIter, 
  int *piCol
){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;

  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
    if( pIter->a>=pIter->b ){
      *piCol = -1;
    }else{
      int iIncr;
      pIter->a += fts5GetVarint32(&pIter->a[0], iIncr);
      *piCol += (iIncr-2);
    }
  }else{
    while( 1 ){
      int dummy;
      if( pIter->a>=pIter->b ){
        *piCol = -1;
        return;
      }
      if( pIter->a[0]==0x01 ) break;
      pIter->a += fts5GetVarint32(pIter->a, dummy);
    }
    pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
  }
}

static int fts5ApiPhraseFirstColumn(
  Fts5Context *pCtx, 
  int iPhrase, 
  Fts5PhraseIter *pIter, 
  int *piCol
){
  int rc = SQLITE_OK;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;

  if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
    int n;
    rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
    if( rc==SQLITE_OK ){
      pIter->b = &pIter->a[n];
      *piCol = 0;
      fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
    }
  }else{
    int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
    pIter->b = &pIter->a[n];
    if( n<=0 ){
      *piCol = -1;
    }else if( pIter->a[0]==0x01 ){
      pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
    }else{
      *piCol = 0;
    }
  }

  return rc;
}


static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
    int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
);

static const Fts5ExtensionApi sFts5Api = {
  2,                            /* iVersion */
................................................................................
  fts5ApiColumnText,
  fts5ApiColumnSize,
  fts5ApiQueryPhrase,
  fts5ApiSetAuxdata,
  fts5ApiGetAuxdata,
  fts5ApiPhraseFirst,
  fts5ApiPhraseNext,
  fts5ApiPhraseFirstColumn,
  fts5ApiPhraseNextColumn,
};

/*
** Implementation of API function xQueryPhrase().
*/
static int fts5ApiQueryPhrase(
  Fts5Context *pCtx,

Changes to ext/fts5/fts5_tcl.c. 

232
233
234
235
236
237
238

239
240
241
242
243
244
245
...
454
455
456
457
458
459
460


























461
462
463
464
465
466
467

    { "xColumnSize",       1, "COL" },                /* 10 */
    { "xQueryPhrase",      2, "PHRASE SCRIPT" },      /* 11 */
    { "xSetAuxdata",       1, "VALUE" },              /* 12 */
    { "xGetAuxdata",       1, "CLEAR" },              /* 13 */
    { "xSetAuxdataInt",    1, "INTEGER" },            /* 14 */
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */

    { 0, 0, 0}
  };

  int rc;
  int iSub = 0;
  F5tApi *p = (F5tApi*)clientData;

................................................................................
        rc = Tcl_EvalObjEx(interp, pScript, 0);
        if( rc==TCL_CONTINUE ) rc = TCL_OK;
        if( rc!=TCL_OK ){
          if( rc==TCL_BREAK ) rc = TCL_OK;
          break;
        }
      }



























      break;
    }

    default: 
      assert( 0 );
      break;








>







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
...
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494

    { "xColumnSize",       1, "COL" },                /* 10 */
    { "xQueryPhrase",      2, "PHRASE SCRIPT" },      /* 11 */
    { "xSetAuxdata",       1, "VALUE" },              /* 12 */
    { "xGetAuxdata",       1, "CLEAR" },              /* 13 */
    { "xSetAuxdataInt",    1, "INTEGER" },            /* 14 */
    { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
    { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
    { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
    { 0, 0, 0}
  };

  int rc;
  int iSub = 0;
  F5tApi *p = (F5tApi*)clientData;

................................................................................
        rc = Tcl_EvalObjEx(interp, pScript, 0);
        if( rc==TCL_CONTINUE ) rc = TCL_OK;
        if( rc!=TCL_OK ){
          if( rc==TCL_BREAK ) rc = TCL_OK;
          break;
        }
      }

      break;
    }

    CASE(17, "xPhraseColumnForeach") {
      int iPhrase;
      int iCol;
      const char *zColvar;
      Tcl_Obj *pScript = objv[4];
      Fts5PhraseIter iter;

      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
      zColvar = Tcl_GetString(objv[3]);

      for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
          iCol>=0;
          p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
      ){
        Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
        rc = Tcl_EvalObjEx(interp, pScript, 0);
        if( rc==TCL_CONTINUE ) rc = TCL_OK;
        if( rc!=TCL_OK ){
          if( rc==TCL_BREAK ) rc = TCL_OK;
          break;
        }
      }

      break;
    }

    default: 
      assert( 0 );
      break;

Changes to ext/fts5/test/fts5_common.tcl. 

35
36
37
38
39
40
41










42
43
44
45
46
47
48
...
122
123
124
125
126
127
128

129
130
131
132
133
134
135

    $cmd xPhraseForeach $i c o {
      lappend res $i.$c.$o
    }
  }

  set res
}











proc fts5_test_columnsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    lappend res [$cmd xColumnSize $i]
  }
  set res
................................................................................
proc fts5_aux_test_functions {db} {
  foreach f {
    fts5_test_columnsize
    fts5_test_columntext
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_poslist2

    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all

    fts5_test_queryphrase
    fts5_test_phrasecount
  } {








>
>
>
>
>
>
>
>
>
>







 







>








35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
...
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

    $cmd xPhraseForeach $i c o {
      lappend res $i.$c.$o
    }
  }

  set res
}

proc fts5_test_collist {cmd} {
  set res [list]

  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    $cmd xPhraseColumnForeach $i c { lappend res $i.$c }
  }

  set res
}

proc fts5_test_columnsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    lappend res [$cmd xColumnSize $i]
  }
  set res
................................................................................
proc fts5_aux_test_functions {db} {
  foreach f {
    fts5_test_columnsize
    fts5_test_columntext
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_poslist2
    fts5_test_collist
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all

    fts5_test_queryphrase
    fts5_test_phrasecount
  } {

Changes to ext/fts5/test/fts5ac.test. 

127
128
129
130
131
132
133
134
135
136

137
138

139
140
141

142
143
144
145
146
147
148
149
150
...
164
165
166
167
168
169
170
171





























172
173
174
175
176
177

178
179
180
181
182
183
184
...
210
211
212
213
214
215
216
217
218
219
220
221





222
223
224
225
226
227
228
...
233
234
235
236
237
238
239
240
241
242
243





244
245
246
247
248
249
250
...
262
263
264
265
266
267
268
269
270
271
272





273
274
275
276
277
278
279
...
283
284
285
286
287
288
289
290
291
292
293





294
295
296
297
298
299
300
...
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349



350
351
352
353
354
355
356
...
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389



390

391
392
393
394
395
396
397
398
399


# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored in the global 
# $::data array (see above), this function returns a list containing one
# element for each match in the dataset. The elements are themselves lists
# formatted as follows:

#
#   <rowid> {<phrase 0 matches> <phrase 1 matches>...}

#
# where each <phrase X matches> element is a list of phrase matches in the
# same form as returned by auxiliary scalar function fts5_test().

#
proc matchdata {bPos expr {bAsc 1}} {

  set tclexpr [db one {
    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
  }]
  set res [list]

  #puts $tclexpr
................................................................................

  if {$bAsc} {
    set res [lsort -integer -increasing -index 0 $res]
  } else {
    set res [lsort -integer -decreasing -index 0 $res]
  }

  return [concat {*}$res]





























}

#
# End of test code
#-------------------------------------------------------------------------



foreach {tn2 sql} {
  1  {}
  2  {BEGIN}
} {
  reset_db
  fts5_aux_test_functions db
................................................................................
    6 "a"
    7 "b"
    8 "c"
    9 "no"
    10 "L O O L V V K"
  } {
    set expr "\"$phrase\""
    set res [matchdata 1 $expr]

    do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res





  }

  #-------------------------------------------------------------------------
  # Test some AND and OR queries.
  #
  foreach {tn expr} {
    1.1 "a   AND b"
................................................................................
    2.1 "a   OR b"
    2.2 "a+b OR c"
    2.3 "d+c OR u"
    2.4 "d+c OR u+d"

    3.1 { a AND b AND c }
  } {
    set res [matchdata 1 $expr]
    do_execsql_test 1.$tn2.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res





  }

  #-------------------------------------------------------------------------
  # Queries on a specific column.
  #
  foreach {tn expr} {
    1.1 "x:a"
................................................................................
    3.4 "{y y}:b"

    4.1 {{"x" "y"}:a}
    4.2 {{"y" x}:a}
    4.3 {{x "x"}:b}
    4.4 {{"y" y}:b}
  } {
    set res [matchdata 1 $expr]
    do_execsql_test 1.$tn2.3.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res





  }

  #-------------------------------------------------------------------------
  # Some NEAR queries.
  #
  foreach {tn expr} {
    1 "NEAR(a b)"
................................................................................
    4 { NEAR(r c, 2) }
    5 { NEAR(r c, 0) }
    6 { NEAR(a b c) }
    7 { NEAR(a b c, 8) }
    8  { x : NEAR(r c) }
    9  { y : NEAR(r c) }
  } {
    set res [matchdata 1 $expr]
    do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res





  }

  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}

  foreach {tn expr tclexpr} {
    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
................................................................................
      14b { a OR b }
      15 { a OR b AND c }
      16 { c AND b OR a }
      17 { c AND (b OR a) }
      18 { c NOT (b OR a) }
      19 { c NOT b OR a AND d }
    } {
      set res [matchdata 0 $expr $bAsc]
      do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
}

do_execsql_test 2.1 {
  SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}




# Some tests for detail=col tables and detail=none.
#
foreach {tn2 sql} {
  1  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
  }
  2  {
................................................................................

  execsql $sql

  do_execsql_test 3.$tn2.0 {
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }


  do_test 3.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
  } {}

  foreach {tn q} {
    1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L"
    11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d"
  } {
    set res [matchdata 1 $q]





    do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q
    } $res
  }

}

finish_test









|
|
|
>

<
>

<
|
>

|







 







|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>






>







 







|

|


>
>
>
>
>







 







|
|


>
>
>
>
>







 







|
|


>
>
>
>
>







 







|
|


>
>
>
>
>







 







|









>
>
>







 







<









|

|
>
>
>

>
|
|








127
128
129
130
131
132
133
134
135
136
137
138

139
140

141
142
143
144
145
146
147
148
149
150
151
...
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
...
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
...
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
...
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
...
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
...
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
...
424
425
426
427
428
429
430

431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456


# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored in the global 
# $::data array (see above), this function returns a list containing two
# elements for each matching row in the dataset. The first element of each
# pair is the rowid. The second is a list of phrase matches, where each 
# phrase match is of the form:
#

#   <phrase-number>.<column-number>.<offset>
#

# The list of phrase matches is in the same format as that returned by the
# fts5_test_poslist() auxiliary scalara function.
#
proc poslist_data {bPos expr {bAsc 1}} {

  set tclexpr [db one {
    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
  }]
  set res [list]

  #puts $tclexpr
................................................................................

  if {$bAsc} {
    set res [lsort -integer -increasing -index 0 $res]
  } else {
    set res [lsort -integer -decreasing -index 0 $res]
  }

  set res [concat {*}$res]
  return $res
}

proc collist_elem_compare {a b} {
  foreach {a1 a2} [split $a .] {}
  foreach {b1 b2} [split $b .] {}

  if {$a1==$b1} {
    return [expr $a2 - $b2]
  }
  return [expr $a1 - $b1]
}

proc poslist2collist {poslist} {
  set res [list]
  foreach h $poslist {
    regexp {(.*)\.[1234567890]+} $h -> cand
    lappend res $cand
  }
  set res [lsort -command collist_elem_compare -unique $res]
  return $res
}

proc collist_data {expr} {
  set res [list]
  foreach {rowid poslist} [poslist_data 1 $expr] {
    lappend res $rowid [poslist2collist $poslist]
  }
  set res
}

#
# End of test code
#-------------------------------------------------------------------------

if 0 {

foreach {tn2 sql} {
  1  {}
  2  {BEGIN}
} {
  reset_db
  fts5_aux_test_functions db
................................................................................
    6 "a"
    7 "b"
    8 "c"
    9 "no"
    10 "L O O L V V K"
  } {
    set expr "\"$phrase\""
    set res [poslist_data 1 $expr]

    do_execsql_test 1.$tn2.1.2.$tn.p.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res

    set res [collist_data $expr]
    do_execsql_test 1.$tn2.1.2.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Test some AND and OR queries.
  #
  foreach {tn expr} {
    1.1 "a   AND b"
................................................................................
    2.1 "a   OR b"
    2.2 "a+b OR c"
    2.3 "d+c OR u"
    2.4 "d+c OR u+d"

    3.1 { a AND b AND c }
  } {
    set res [poslist_data 1 $expr]
    do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res

    set res [collist_data $expr]
    do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Queries on a specific column.
  #
  foreach {tn expr} {
    1.1 "x:a"
................................................................................
    3.4 "{y y}:b"

    4.1 {{"x" "y"}:a}
    4.2 {{"y" x}:a}
    4.3 {{x "x"}:b}
    4.4 {{"y" y}:b}
  } {
    set res [poslist_data 1 $expr]
    do_execsql_test 1.$tn2.3.$tn.p.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res

    set res [collist_data $expr]
    do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  #-------------------------------------------------------------------------
  # Some NEAR queries.
  #
  foreach {tn expr} {
    1 "NEAR(a b)"
................................................................................
    4 { NEAR(r c, 2) }
    5 { NEAR(r c, 0) }
    6 { NEAR(a b c) }
    7 { NEAR(a b c, 8) }
    8  { x : NEAR(r c) }
    9  { y : NEAR(r c) }
  } {
    set res [poslist_data 1 $expr]
    do_execsql_test 1.$tn2.4.1.$tn.p.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
    } $res

    set res [collist_data $expr]
    do_execsql_test 1.$tn2.4.1.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
    } $res
  }

  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}

  foreach {tn expr tclexpr} {
    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
................................................................................
      14b { a OR b }
      15 { a OR b AND c }
      16 { c AND b OR a }
      17 { c AND (b OR a) }
      18 { c NOT (b OR a) }
      19 { c NOT b OR a AND d }
    } {
      set res [poslist_data 0 $expr $bAsc]
      do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res
    }
  }
}

do_execsql_test 2.1 {
  SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}

}
#set data [lrange $data 0 5]

# Some tests for detail=col tables and detail=none.
#
foreach {tn2 sql} {
  1  {
    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
  }
  2  {
................................................................................

  execsql $sql

  do_execsql_test 3.$tn2.0 {
    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
  }


  do_test 3.$tn2.1.1 {
    foreach {id x y} $data {
      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
    }
    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
  } {}

  foreach {tn q} {
    1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L"
    11 "o a" 12 "c AND d" 13 "o OR a" 14 "c OR d"
  } {
    set res [poslist_data 1 $q]
    do_execsql_test 3.$tn2.1.2.$tn.p.[llength $res] { 
      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q
    } $res

    set res [collist_data $q]
    do_execsql_test 3.$tn2.1.2.$tn.c.[llength $res] { 
      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $q
    } $res
  }

}

finish_test

This page was generated in about 0.021s by Fossil 2.11 [4dbd4e33c4] 2020-02-16 15:25:00