/ Check-in [fd217fdb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Rationalize and extend some fts5 test code. Fix bugs revealed by the same.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: fd217fdb396cf22278a496de0e9dec7902ba8b1f
User & Date: dan 2016-01-05 21:04:35
Context
2016-01-06
10:17
Fix a problem with the xPhraseFirstColumn() API in detail=col mode. Remove some unused test code from fts5ac.test. check-in: a4e55f73 user: dan tags: fts5-offsets
2016-01-05
21:04
Rationalize and extend some fts5 test code. Fix bugs revealed by the same. check-in: fd217fdb user: dan tags: fts5-offsets
2016-01-04
19:12
Fix an OOM handling problem in fts5 detail=none and detail=col modes. Also a bug in the xInst() API when used with detail=col and column filtering expressions. Update the matchinfo() test function so that 'b' is fast with detail=col tables. check-in: ee38f9df user: dan tags: fts5-offsets
Changes
Hide Diffs Side-by-Side Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5Int.h.

   639    639   int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   640    640   
   641    641   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   642    642   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   643    643   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
   644    644   
   645    645   typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
   646         -Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
          646  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
   647    647   int sqlite3Fts5ExprPopulatePoslists(
   648    648       Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
   649    649   );
   650    650   void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
          651  +void sqlite3Fts5ExprClearEof(Fts5Expr*);
   651    652   
   652    653   int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
   653    654   
   654    655   int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   655    656   
   656    657   /*******************************************
   657    658   ** The fts5_expr.c API above this point is used by the other hand-written

Changes to ext/fts5/fts5_expr.c.

  1214   1214             }
  1215   1215             assert( rc!=SQLITE_OK || cmp<=0 );
  1216   1216             if( cmp || p2->bNomatch ) break;
  1217   1217             rc = fts5ExprNodeNext(pExpr, p1, 0, 0);
  1218   1218           }
  1219   1219           pNode->bEof = p1->bEof;
  1220   1220           pNode->iRowid = p1->iRowid;
         1221  +        if( p1->bEof ){
         1222  +          fts5ExprNodeZeroPoslist(p2);
         1223  +        }
  1221   1224           break;
  1222   1225         }
  1223   1226       }
  1224   1227     }
  1225   1228     return rc;
  1226   1229   }
  1227   1230   
................................................................................
  2256   2259     }
  2257   2260     return nRet;
  2258   2261   }
  2259   2262   
  2260   2263   struct Fts5PoslistPopulator {
  2261   2264     Fts5PoslistWriter writer;
  2262   2265     int bOk;                        /* True if ok to populate */
         2266  +  int bMiss;
  2263   2267   };
  2264   2268   
  2265         -Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
         2269  +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
  2266   2270     Fts5PoslistPopulator *pRet;
  2267   2271     pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
  2268   2272     if( pRet ){
  2269   2273       int i;
  2270   2274       memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
  2271   2275       for(i=0; i<pExpr->nPhrase; i++){
  2272   2276         Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
         2277  +      Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
  2273   2278         assert( pExpr->apExprPhrase[i]->nTerm==1 );
         2279  +      if( bLive && 
         2280  +          (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
         2281  +      ){
         2282  +        pRet[i].bMiss = 1;
         2283  +      }else{
  2274   2284         pBuf->n = 0;
  2275   2285       }
  2276   2286     }
         2287  +  }
  2277   2288     return pRet;
  2278   2289   }
  2279   2290   
  2280   2291   struct Fts5ExprCtx {
  2281   2292     Fts5Expr *pExpr;
  2282   2293     Fts5PoslistPopulator *aPopulator;
  2283   2294     i64 iOff;
................................................................................
  2337   2348     int i;
  2338   2349     Fts5ExprCtx sCtx;
  2339   2350     sCtx.pExpr = pExpr;
  2340   2351     sCtx.aPopulator = aPopulator;
  2341   2352     sCtx.iOff = (((i64)iCol) << 32) - 1;
  2342   2353   
  2343   2354     for(i=0; i<pExpr->nPhrase; i++){
  2344         -    Fts5Colset *pColset = pExpr->apExprPhrase[i]->pNode->pNear->pColset;
  2345         -    if( pColset && 0==fts5ExprColsetTest(pColset, iCol) ){
         2355  +    Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
         2356  +    Fts5Colset *pColset = pNode->pNear->pColset;
         2357  +    if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) 
         2358  +     || aPopulator[i].bMiss
         2359  +    ){
  2346   2360         aPopulator[i].bOk = 0;
  2347   2361       }else{
  2348   2362         aPopulator[i].bOk = 1;
  2349   2363       }
  2350   2364     }
  2351   2365   
  2352   2366     return sqlite3Fts5Tokenize(pConfig, 
................................................................................
  2408   2422             return 0;
  2409   2423           }
  2410   2424           return 1;
  2411   2425         }
  2412   2426       }
  2413   2427     }
  2414   2428   }
         2429  +
  2415   2430   void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  2416   2431     fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
  2417   2432   }
         2433  +
         2434  +static void fts5ExprClearEof(Fts5ExprNode *pNode){
         2435  +  int i;
         2436  +  for(i=0; i<pNode->nChild; i++){
         2437  +    fts5ExprClearEof(pNode->apChild[i]);
         2438  +  }
         2439  +  pNode->bEof = 0;
         2440  +}
         2441  +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
         2442  +  fts5ExprClearEof(pExpr->pRoot);
         2443  +}
  2418   2444   
  2419   2445   /*
  2420   2446   ** This function is only called for detail=columns tables. 
  2421   2447   */
  2422   2448   int sqlite3Fts5ExprPhraseCollist(
  2423   2449     Fts5Expr *pExpr, 
  2424   2450     int iPhrase, 

Changes to ext/fts5/fts5_main.c.

  1175   1175       assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
  1176   1176       assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
  1177   1177       assert( pCsr->iLastRowid==LARGEST_INT64 );
  1178   1178       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
  1179   1179       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1180   1180       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1181   1181       rc = fts5CursorFirst(pTab, pCsr, bDesc);
         1182  +    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  1182   1183     }else if( pMatch ){
  1183   1184       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1184   1185       if( zExpr==0 ) zExpr = "";
  1185   1186   
  1186   1187       rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1187   1188       if( rc==SQLITE_OK ){
  1188   1189         if( zExpr[0]=='*' ){
................................................................................
  1684   1685     Fts5Cursor *pCsr, 
  1685   1686     int iPhrase, 
  1686   1687     const u8 **pa,
  1687   1688     int *pn
  1688   1689   ){
  1689   1690     Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
  1690   1691     int rc = SQLITE_OK;
         1692  +  int bLive = (pCsr->pSorter==0);
  1691   1693   
  1692   1694     if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
  1693   1695   
  1694   1696       if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
  1695   1697         Fts5PoslistPopulator *aPopulator;
  1696   1698         int i;
  1697         -      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
         1699  +      aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
  1698   1700         if( aPopulator==0 ) rc = SQLITE_NOMEM;
  1699   1701         for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
  1700   1702           int n; const char *z;
  1701   1703           rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
  1702   1704           if( rc==SQLITE_OK ){
  1703   1705             rc = sqlite3Fts5ExprPopulatePoslists(
  1704   1706                 pConfig, pCsr->pExpr, aPopulator, i, z, n
  1705   1707             );
  1706   1708           }
  1707   1709         }
  1708   1710         sqlite3_free(aPopulator);
         1711  +
  1709   1712         if( pCsr->pSorter ){
  1710   1713           sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
  1711   1714         }
  1712   1715       }
  1713   1716       CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
  1714   1717     }
  1715   1718   

Changes to ext/fts5/test/fts5_common.tcl.

   386    386     }
   387    387   }
   388    388   proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
   389    389   proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
   390    390   proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
   391    391   
   392    392   
          393  +#-------------------------------------------------------------------------
          394  +# Convert a poslist of the type returned by fts5_test_poslist() to a 
          395  +# collist as returned by fts5_test_collist().
          396  +#
          397  +proc fts5_poslist2collist {poslist} {
          398  +  set res [list]
          399  +  foreach h $poslist {
          400  +    regexp {(.*)\.[1234567890]+} $h -> cand
          401  +    lappend res $cand
          402  +  }
          403  +  set res [lsort -command fts5_collist_elem_compare -unique $res]
          404  +  return $res
          405  +}
          406  +
          407  +# Comparison function used by fts5_poslist2collist to sort collist entries.
          408  +#
          409  +proc fts5_collist_elem_compare {a b} {
          410  +  foreach {a1 a2} [split $a .] {}
          411  +  foreach {b1 b2} [split $b .] {}
          412  +
          413  +  if {$a1==$b1} { return [expr $a2 - $b2] }
          414  +  return [expr $a1 - $b1]
          415  +}
          416  +
          417  +
          418  +#--------------------------------------------------------------------------
          419  +# Construct and return a tcl list equivalent to that returned by the SQL
          420  +# query executed against database handle [db]:
          421  +#
          422  +#   SELECT 
          423  +#     rowid, 
          424  +#     fts5_test_poslist($tbl),
          425  +#     fts5_test_collist($tbl) 
          426  +#   FROM $tbl('$expr')
          427  +#   ORDER BY rowid $order;
          428  +#
          429  +proc fts5_query_data {expr tbl {order ASC}} {
          430  +
          431  +  # Figure out the set of columns in the FTS5 table. This routine does
          432  +  # not handle tables with UNINDEXED columns, but if it did, it would
          433  +  # have to be here.
          434  +  db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
          435  +
          436  +  set cols ""
          437  +  foreach e $lCols { append cols ", '$e'" }
          438  +  set tclexpr [db one [subst -novar {
          439  +    SELECT fts5_expr_tcl( $expr, 'nearset $cols -pc ::pc' [set cols] )
          440  +  }]]
          441  +
          442  +  set res [list]
          443  +  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
          444  +    set cols [list]
          445  +    foreach col $lCols { lappend cols $x($col) }
          446  +    
          447  +    set ::pc 0
          448  +    set rowdata [eval $tclexpr]
          449  +    if {$rowdata != ""} { 
          450  +      lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
          451  +    }
          452  +  }
          453  +
          454  +  set res
          455  +}
          456  +
          457  +#-------------------------------------------------------------------------
          458  +# Similar to [fts5_query_data], but omit the collist field.
          459  +#
          460  +proc fts5_poslist_data {expr tbl {order ASC}} {
          461  +  set res [list]
          462  +  foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order] {
          463  +    lappend res $rowid $poslist
          464  +  }
          465  +  set res
          466  +}
          467  +
          468  +#-------------------------------------------------------------------------
          469  +#
          470  +proc nearset_rf {aCol args} {
          471  +  set idx [lsearch -exact $args --]
          472  +  if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
          473  +    set ::expr_not_ok 1
          474  +  }
          475  +  list
          476  +}
          477  +
          478  +proc nearset_rc {aCol args} {
          479  +  nearset_rf $aCol {*}$args
          480  +  if {[lsearch $args -col]>=0} { 
          481  +    set ::expr_not_ok 1
          482  +  }
          483  +  list
          484  +}
          485  +
          486  +proc fts5_expr_ok {expr tbl} {
          487  +
          488  +  if {![detail_is_full]} {
          489  +    set nearset "nearset_rc"
          490  +    if {[detail_is_col]} { set nearset "nearset_rf" }
          491  +
          492  +    set ::expr_not_ok 0
          493  +    db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
          494  +
          495  +    set cols ""
          496  +    foreach e $lCols { append cols ", '$e'" }
          497  +    set ::pc 0
          498  +    set tclexpr [db one [subst -novar {
          499  +      SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
          500  +    }]]
          501  +    eval $tclexpr
          502  +    if {$::expr_not_ok} { return 0 }
          503  +  }
   393    504   
          505  +  return 1
          506  +}
   394    507   

Changes to ext/fts5/test/fts5ac.test.

   207    207   # End of test code
   208    208   #-------------------------------------------------------------------------
   209    209   
   210    210   foreach {tn2 sql} {
   211    211     1  {}
   212    212     2  {BEGIN}
   213    213   } {
          214  +if {[detail_is_full]} continue
   214    215     reset_db
   215    216     fts5_aux_test_functions db
   216    217   
   217    218     do_execsql_test 1.$tn2.0 {
   218    219       CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
   219    220       INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
   220    221     }
................................................................................
   224    225     do_test 1.$tn2.1.1 {
   225    226       foreach {id x y} $data {
   226    227         execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
   227    228       }
   228    229       execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
   229    230     } {}
   230    231   
   231         -  #-------------------------------------------------------------------------
   232         -  # The following work with all detail= modes.
   233         -  #
   234         -  foreach {tn expr} {
   235         -    1 "a   AND b"
   236         -    2 "a   OR b"
   237         -    3 "o"
   238         -    4 "b q"
   239         -    5 "e a e"
   240         -    6 "m d g q q b k b w f q q p p"
   241         -    7 "l o o l v v k"
   242         -    8 "a"
   243         -    9 "b"
   244         -    10 "c"
   245         -    11 "no"
   246         -    12 "L O O L V V K"
   247         -    13 "a AND b AND c"
   248         -  } {
   249         -    set res [poslist_data 1 $expr]
   250         -    do_execsql_test 1.$tn2.1.$tn.p.[llength $res] { 
   251         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   252         -    } $res
   253         -
   254         -    set res [collist_data $expr]
   255         -    do_execsql_test 1.$tn2.1.$tn.c.[llength $res] { 
   256         -      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
   257         -    } $res
   258         -  }
   259         -
   260         -  if {[detail_is_none]} continue
   261    232   
   262    233     #-------------------------------------------------------------------------
   263         -  # Queries on a specific column.
   264    234     #
   265         -  foreach {tn expr} {
   266         -    1.1 "x:a"
   267         -    1.2 "y:a"
   268         -    1.3 "x:b"
   269         -    1.4 "y:b"
   270         -    2.1 "{x}:a"
   271         -    2.2 "{y}:a"
   272         -    2.3 "{x}:b"
   273         -    2.4 "{y}:b"
   274         -
   275         -    3.1 "{x y}:a"
   276         -    3.2 "{y x}:a"
   277         -    3.3 "{x x}:b"
   278         -    3.4 "{y y}:b"
   279         -
   280         -    4.1 {{"x" "y"}:a}
   281         -    4.2 {{"y" x}:a}
   282         -    4.3 {{x "x"}:b}
   283         -    4.4 {{"y" y}:b}
   284         -  } {
   285         -    set res [poslist_data 1 $expr]
   286         -    do_execsql_test 1.$tn2.4.$tn.p.[llength $res] { 
   287         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   288         -    } $res
   289         -
   290         -    set res [collist_data $expr]
   291         -    do_execsql_test 1.$tn2.4.$tn.c.[llength $res] { 
   292         -      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
   293         -    } $res
   294         -  }
   295         -
   296         -  if {[detail_is_col]} continue
   297         -
   298         -  #-------------------------------------------------------------------------
   299         -  # Test phrase queries.
   300         -  #
   301         -  foreach {tn expr} {
   302         -    1 "b + q"
   303         -    2 "e + a + e"
   304         -    3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
   305         -    4 "l + o + o + l + v + v + k"
   306         -    5 "L + O + O + L + V + V + K"
   307         -  } {
   308         -    set res [poslist_data 1 $expr]
   309         -
   310         -    do_execsql_test 1.$tn2.2.$tn.p.[llength $res] { 
   311         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   312         -    } $res
   313         -
   314         -    set res [collist_data $expr]
   315         -    do_execsql_test 1.$tn2.2.$tn.c.[llength $res] { 
   316         -      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
   317         -    } $res
   318         -  }
   319         -
   320         -  #-------------------------------------------------------------------------
   321         -  # Test some AND and OR queries.
   322         -  #
   323         -  foreach {tn expr} {
   324         -    1.1 "a+b AND c"
   325         -    1.2 "d+c AND u"
   326         -    1.3 "d+c AND u+d"
   327         -
   328         -    2.1 "a+b OR c"
   329         -    2.2 "d+c OR u"
   330         -    2.3 "d+c OR u+d"
   331         -  } {
   332         -    set res [poslist_data 1 $expr]
   333         -    do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { 
   334         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   335         -    } $res
   336         -
   337         -    set res [collist_data $expr]
   338         -    do_execsql_test 1.$tn2.3.$tn.c.[llength $res] { 
   339         -      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
   340         -    } $res
   341         -  }
   342         -
   343         -  #-------------------------------------------------------------------------
   344         -  # Some NEAR queries.
   345         -  #
   346         -  foreach {tn expr} {
   347         -    1 "NEAR(a b)"
   348         -    2 "NEAR(r c)"
   349         -    2 { NEAR(r c, 5) }
   350         -    3 { NEAR(r c, 3) }
   351         -    4 { NEAR(r c, 2) }
   352         -    5 { NEAR(r c, 0) }
   353         -    6 { NEAR(a b c) }
   354         -    7 { NEAR(a b c, 8) }
   355         -    8  { x : NEAR(r c) }
   356         -    9  { y : NEAR(r c) }
   357         -  } {
   358         -    set res [poslist_data 1 $expr]
   359         -    do_execsql_test 1.$tn2.5.1.$tn.p.[llength $res] { 
   360         -      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   361         -    } $res
   362         -
   363         -    set res [collist_data $expr]
   364         -    do_execsql_test 1.$tn2.5.1.$tn.c.[llength $res] { 
   365         -      SELECT rowid, fts5_test_collist(xx) FROM xx WHERE xx match $expr
   366         -    } $res
   367         -  }
   368         -
   369         -  do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
   370         -  do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}
   371         -
   372         -  foreach {tn expr tclexpr} {
   373         -    1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
   374         -  } {
   375         -    do_execsql_test 1.$tn2.6.$tn {
   376         -      SELECT fts5_expr_tcl($expr, 'N $x')
   377         -    } [list $tclexpr]
          235  +  do_execsql_test 1.$tn2.integrity {
          236  +    INSERT INTO xx(xx) VALUES('integrity-check');
   378    237     }
   379    238   
   380    239     #-------------------------------------------------------------------------
   381    240     #
   382         -  do_execsql_test 1.$tn2.7.integrity {
   383         -    INSERT INTO xx(xx) VALUES('integrity-check');
   384         -  }
   385         -  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
   386         -  foreach {bAsc sql} {
   387         -    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
   388         -    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
   389         -  } {
   390    241       foreach {tn expr} {
   391         -      0.1 x
   392         -      1 { NEAR(r c) }
   393         -      2 { NEAR(r c, 5) }
   394         -      3 { NEAR(r c, 3) }
   395         -      4 { NEAR(r c, 2) }
   396         -      5 { NEAR(r c, 0) }
   397         -      6 { NEAR(a b c) }
   398         -      7 { NEAR(a b c, 8) }
   399         -      8  { x : NEAR(r c) }
   400         -      9  { y : NEAR(r c) }
   401         -      10 { x : "r c" }
   402         -      11 { y : "r c" }
   403         -      12 { a AND b }
   404         -      13 { a AND b AND c }
   405         -      14a { a }
   406         -      14b { a OR b }
   407         -      15 { a OR b AND c }
   408         -      16 { c AND b OR a }
   409         -      17 { c AND (b OR a) }
   410         -      18 { c NOT (b OR a) }
   411         -      19 { c NOT b OR a AND d }
          242  +    1.2 "a   OR b"
          243  +    1.1 "a   AND b"
          244  +    1.3 "o"
          245  +    1.4 "b q"
          246  +    1.5 "e a e"
          247  +    1.6 "m d g q q b k b w f q q p p"
          248  +    1.7 "l o o l v v k"
          249  +    1.8 "a"
          250  +    1.9 "b"
          251  +    1.10 "c"
          252  +    1.11 "no"
          253  +    1.12 "L O O L V V K"
          254  +    1.13 "a AND b AND c"
          255  +    1.14 "x:a"
          256  +
          257  +    2.1 "x:a"
          258  +    2.2 "y:a"
          259  +    2.3 "x:b"
          260  +    2.4 "y:b"
          261  +
          262  +    3.1 "{x}:a"
          263  +    3.2 "{y}:a"
          264  +    3.3 "{x}:b"
          265  +    3.4 "{y}:b"
          266  +
          267  +    4.1 "{x y}:a"
          268  +    4.2 "{y x}:a"
          269  +    4.3 "{x x}:b"
          270  +    4.4 "{y y}:b"
          271  +
          272  +    5.1 {{"x" "y"}:a}
          273  +    5.2 {{"y" x}:a}
          274  +    5.3 {{x "x"}:b}
          275  +    5.4 {{"y" y}:b}
          276  +
          277  +    6.1 "b + q"
          278  +    6.2 "e + a + e"
          279  +    6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
          280  +    6.4 "l + o + o + l + v + v + k"
          281  +    6.5 "L + O + O + L + V + V + K"
          282  +
          283  +    7.1 "a+b AND c"
          284  +    7.2 "d+c AND u"
          285  +    7.3 "d+c AND u+d"
          286  +    7.4 "a+b OR c"
          287  +    7.5 "d+c OR u"
          288  +    7.6 "d+c OR u+d"
          289  +
          290  +    8.1 "NEAR(a b)"
          291  +    8.2 "NEAR(r c)"
          292  +    8.2 { NEAR(r c, 5) }
          293  +    8.3 { NEAR(r c, 3) }
          294  +    8.4 { NEAR(r c, 2) }
          295  +    8.5 { NEAR(r c, 0) }
          296  +    8.6 { NEAR(a b c) }
          297  +    8.7 { NEAR(a b c, 8) }
          298  +    8.8  { x : NEAR(r c) }
          299  +    8.9  { y : NEAR(r c) }
          300  +
          301  +    9.1 { NEAR(r c) }
          302  +    9.2 { NEAR(r c, 5) }
          303  +    9.3 { NEAR(r c, 3) }
          304  +    9.4 { NEAR(r c, 2) }
          305  +    9.5 { NEAR(r c, 0) }
          306  +    9.6 { NEAR(a b c) }
          307  +    9.7 { NEAR(a b c, 8) }
          308  +    9.8  { x : NEAR(r c) }
          309  +    9.9  { y : NEAR(r c) }
          310  +    9.10 { x : "r c" }
          311  +    9.11 { y : "r c" }
          312  +    9.12 { a AND b }
          313  +    9.13 { a AND b AND c }
          314  +    9.14a { a }
          315  +    9.14b { a OR b }
          316  +    9.15 { a OR b AND c }
          317  +    9.16 { c AND b OR a }
          318  +    9.17 { c AND (b OR a) }
          319  +    9.18 { c NOT (b OR a) }
          320  +    9.19 { (c NOT b) OR (a AND d) }
   412    321       } {
   413         -      set res [poslist_data 0 $expr $bAsc]
   414         -      do_execsql_test 1.$tn2.7.$bAsc.$tn.[llength $res] $sql $res
          322  +
          323  +    if {[fts5_expr_ok $expr xx]==0} {
          324  +      do_test 1.$tn2.$tn.OMITTED { list } [list]
          325  +      continue
   415    326       }
          327  +
          328  +    set res [fts5_query_data $expr xx]
          329  +    do_execsql_test 1.$tn2.$tn.[llength $res].asc {
          330  +      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
          331  +      FROM xx WHERE xx match $expr
          332  +    } $res
          333  +
          334  +
          335  +    set res [fts5_query_data $expr xx DESC]
          336  +    do_execsql_test 1.$tn2.$tn.[llength $res].desc {
          337  +      SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
          338  +      FROM xx WHERE xx match $expr ORDER BY 1 DESC
          339  +    } $res
   416    340     }
   417    341   }
   418    342   
   419    343   }
   420    344   
   421    345   do_execsql_test 2.1 {
   422    346     SELECT fts5_expr_tcl('a AND b');
   423    347   } {{AND [nearset -- {a}] [nearset -- {b}]}}
   424    348   
   425         -finish_test
          349  +do_test 2.2.1  { nearset {{a b c}} -- a } {0.0.0}
          350  +do_test 2.2.2  { nearset {{a b c}} -- c } {0.0.2}
          351  +
          352  +foreach {tn expr tclexpr} {
          353  +  1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
          354  +} {
          355  +  do_execsql_test 2.3.$tn {
          356  +    SELECT fts5_expr_tcl($expr, 'N $x')
          357  +  } [list $tclexpr]
          358  +}
   426    359   
   427    360   finish_test
   428    361   

Changes to ext/fts5/test/fts5auto.test.

    18     18   
    19     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    20     20   ifcapable !fts5 {
    21     21     finish_test
    22     22     return
    23     23   }
    24     24   
    25         -
    26     25   set data {
    27     26       -4026076
    28     27       {n x w k b p x b n t t d s}     {f j j s p j o}               
    29     28       {w v i y r}                     {i p y s}                     
    30     29       {a o q v e n q r}               {q v g u c y a z y}           
    31     30       3995120
    32     31       {c}                             {e e w d t}                   
................................................................................
   228    227   
   229    228   do_execsql_test 1.0 {
   230    229     CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f);
   231    230   } {}
   232    231   
   233    232   fts5_aux_test_functions db
   234    233   
   235         -proc matchdata {expr tbl collist {order ASC}} {
   236         -
   237         -  set cols ""
   238         -  foreach e $collist {
   239         -    append cols ", '$e'"
   240         -  }
   241         -
   242         -  set tclexpr [db one [subst -novar {
   243         -    SELECT fts5_expr_tcl(
   244         -      $expr, 'nearset $cols -pc ::pc' [set cols]
   245         -    )
   246         -  }]]
   247         -  set res [list]
   248         -
   249         -  db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
   250         -    set cols [list]
   251         -    foreach col $x(*) {
   252         -      if {$col != "rowid"} { lappend cols $x($col) }
   253         -    }
   254         -    # set cols [list $a $b $c $d $e $f]
   255         -    set ::pc 0
   256         -    set rowdata [eval $tclexpr]
   257         -    if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
   258         -  }
   259         -
   260         -  set res
   261         -}
   262         -
   263         -proc do_auto_test {tn tbl cols expr} { 
          234  +proc do_auto_test {tn tbl expr} {
   264    235     foreach order {asc desc} {
   265         -    set res [matchdata $expr $tbl $cols $order]
          236  +    set res [fts5_poslist_data $expr $tbl $order]
   266    237       set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
   267    238   
   268    239       set ::autotest_expr $expr
   269    240       do_execsql_test $testname [subst -novar {
   270    241         SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] 
   271    242         WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
   272    243       }] $res
   273    244     }
   274         -
   275         -
   276    245   }
   277    246   
   278    247   #-------------------------------------------------------------------------
   279    248   #
   280    249   
   281    250   for {set fold 0} {$fold < 3} {incr fold} {
   282    251     switch $fold {
................................................................................
   328    297       B.4 { a OR (b AND {a b c}:c) }
   329    298       B.5 { a OR "b c" }
   330    299       B.6 { a OR b OR c }
   331    300   
   332    301       C.1 { a OR (b AND "b c") }
   333    302       C.2 { a OR (b AND "z c") }
   334    303     } {
   335         -    do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
          304  +    do_auto_test 3.$fold.$tn tt $expr
   336    305     }
   337    306   }
   338    307   
   339    308   proc replace_elems {list args} {
   340    309     set ret $list
   341    310     foreach {idx elem} $args {
   342    311       set ret [lreplace $ret $idx $idx $elem]
................................................................................
   362    331     1 x    
   363    332     2 y    
   364    333     3 z
   365    334   
   366    335     4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
   367    336     7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
   368    337     10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
   369         -
   370         -
   371    338   } {
   372         -  do_auto_test 4.$tn yy {c1 c2 c3} $expr
          339  +  do_auto_test 4.$tn yy $expr
   373    340   }
   374    341   
   375    342   
   376    343   
   377    344   finish_test
   378    345   

Changes to ext/fts5/test/fts5simple.test.

    14     14   set testprefix fts5simple
    15     15   
    16     16   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    17     17   ifcapable !fts5 {
    18     18     finish_test
    19     19     return
    20     20   }
           21  +
           22  + if 1 {
    21     23   
    22     24   #-------------------------------------------------------------------------
    23     25   #
    24     26   set doc "x x [string repeat {y } 50]z z"
    25     27   do_execsql_test 1.0 {
    26     28     CREATE VIRTUAL TABLE t1 USING fts5(x);
    27     29     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
................................................................................
   356    358     CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);
   357    359     INSERT INTO x2 VALUES('ab');
   358    360   }
   359    361   
   360    362   do_execsql_test 15.1 {
   361    363     INSERT INTO x2(x2) VALUES('integrity-check');
   362    364   }
          365  +
          366  +}
          367  +
          368  +#-------------------------------------------------------------------------
          369  +foreach_detail_mode $testprefix {
          370  +  reset_db
          371  +  fts5_aux_test_functions db
          372  +  do_execsql_test 16.0 {
          373  +    CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
          374  +    INSERT INTO x3 VALUES('a b c d e f');
          375  +  }
          376  +  do_execsql_test 16.1 {
          377  +    SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
          378  +  } {2.0.2}
          379  +
          380  +  do_execsql_test 16.1 {
          381  +    SELECT fts5_test_poslist(x3) FROM x3('a OR c');
          382  +  } {{0.0.0 1.0.2}}
          383  +}
   363    384   
   364    385   finish_test
   365    386