/ Check-in [64ca1a83]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an FTS5 problem (segfault or incorrect query results) with "... MATCH 'x OR y' ORDER BY rank" queries when either token 'x' or 'y' is completely absent from the dataset.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 64ca1a835a89fd211078d2cd8f9b649e89be528d
User & Date: dan 2016-05-30 08:28:21
References
2016-07-21
18:02
Add extra test cases to verify the fix in [64ca1a835]. check-in: bf98a2de user: dan tags: trunk
Context
2016-05-31
16:22
Add the "csv" virtual table for reading CSV files, as an extension in the ext/misc/ subfolder. check-in: 00d3570c user: drh tags: trunk
2016-05-30
08:28
Fix an FTS5 problem (segfault or incorrect query results) with "... MATCH 'x OR y' ORDER BY rank" queries when either token 'x' or 'y' is completely absent from the dataset. check-in: 64ca1a83 user: dan tags: trunk
2016-05-28
17:45
Remove an unnecessary malloc from the vfsstat extension. check-in: 24f258c2 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   682    682   
   683    683   typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
   684    684   Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
   685    685   int sqlite3Fts5ExprPopulatePoslists(
   686    686       Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
   687    687   );
   688    688   void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
   689         -void sqlite3Fts5ExprClearEof(Fts5Expr*);
   690    689   
   691    690   int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
   692    691   
   693    692   int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
   694    693   
   695    694   /*******************************************
   696    695   ** The fts5_expr.c API above this point is used by the other hand-written

Changes to ext/fts5/fts5_expr.c.

  2613   2613     return 1;
  2614   2614   }
  2615   2615   
  2616   2616   void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  2617   2617     fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
  2618   2618   }
  2619   2619   
  2620         -static void fts5ExprClearEof(Fts5ExprNode *pNode){
  2621         -  int i;
  2622         -  for(i=0; i<pNode->nChild; i++){
  2623         -    fts5ExprClearEof(pNode->apChild[i]);
  2624         -  }
  2625         -  pNode->bEof = 0;
  2626         -}
  2627         -void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
  2628         -  fts5ExprClearEof(pExpr->pRoot);
  2629         -}
  2630         -
  2631   2620   /*
  2632   2621   ** This function is only called for detail=columns tables. 
  2633   2622   */
  2634   2623   int sqlite3Fts5ExprPhraseCollist(
  2635   2624     Fts5Expr *pExpr, 
  2636   2625     int iPhrase, 
  2637   2626     const u8 **ppCollist, 

Changes to ext/fts5/fts5_main.c.

  1182   1182       assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
  1183   1183       assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
  1184   1184       assert( pCsr->iLastRowid==LARGEST_INT64 );
  1185   1185       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
  1186   1186       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1187   1187       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1188   1188       rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1189         -    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  1190   1189     }else if( pMatch ){
  1191   1190       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1192   1191       if( zExpr==0 ) zExpr = "";
  1193   1192   
  1194   1193       rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1195   1194       if( rc==SQLITE_OK ){
  1196   1195         if( zExpr[0]=='*' ){

Changes to ext/fts5/test/fts5rank.test.

    87     87   } {1 3 2}
    88     88   
    89     89   do_test 2.7 {
    90     90     execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
    91     91   } {1 3 2}
    92     92   
    93     93   
           94  +#--------------------------------------------------------------------------
           95  +# At one point there was a problem with queries such as:
           96  +#
           97  +#   ... MATCH 'x OR y' ORDER BY rank;
           98  +#
           99  +# if there were zero occurrences of token 'y' in the dataset. The
          100  +# following tests verify that that problem has been addressed.
          101  +#
          102  +foreach_detail_mode $::testprefix {
          103  +  do_execsql_test 3.0 {
          104  +    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
          105  +    INSERT INTO y1 VALUES('test xyz');
          106  +    INSERT INTO y1 VALUES('test test xyz test');
          107  +    INSERT INTO y1 VALUES('test test xyz');
          108  +  }
          109  +
          110  +  do_execsql_test 3.1 {
          111  +    SELECT rowid FROM y1('test OR tset');
          112  +  } {1 2 3}
          113  +
          114  +  do_execsql_test 3.2 {
          115  +    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
          116  +  } {2 3 1}
    94    117   
          118  +  do_execsql_test 3.3 {
          119  +    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
          120  +  } {2 3 1}
    95    121   
    96         -
          122  +  do_execsql_test 3.4 {
          123  +    SELECT rowid FROM y1('test OR tset') ORDER BY rank
          124  +  } {2 3 1}
          125  +}
    97    126   
    98    127   
    99    128   finish_test
   100    129