SQLite

Check-in [64ca1a835a]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an FTS5 problem (segfault or incorrect query results) with "... MATCH 'x OR y' ORDER BY rank" queries when either token 'x' or 'y' is completely absent from the dataset.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 64ca1a835a89fd211078d2cd8f9b649e89be528d
User & Date: dan 2016-05-30 08:28:21.370
References
2016-07-21
18:02
Add extra test cases to verify the fix in [64ca1a835]. (check-in: bf98a2de7e user: dan tags: trunk)
Context
2016-05-31
16:22
Add the "csv" virtual table for reading CSV files, as an extension in the ext/misc/ subfolder. (check-in: 00d3570c8b user: drh tags: trunk)
2016-05-30
08:28
Fix an FTS5 problem (segfault or incorrect query results) with "... MATCH 'x OR y' ORDER BY rank" queries when either token 'x' or 'y' is completely absent from the dataset. (check-in: 64ca1a835a user: dan tags: trunk)
2016-05-28
17:45
Remove an unnecessary malloc from the vfsstat extension. (check-in: 24f258c239 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5Int.h.
︙︙
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696


typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
void sqlite3Fts5ExprClearEof(Fts5Expr*);

int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written








<








682
683
684
685
686
687
688

689
690
691
692
693
694
695


typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
int sqlite3Fts5ExprPopulatePoslists(
    Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);


int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);

int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);

/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
︙︙
Changes to ext/fts5/fts5_expr.c.
︙︙
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637

  return 1;
}

void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
}

static void fts5ExprClearEof(Fts5ExprNode *pNode){
  int i;
  for(i=0; i<pNode->nChild; i++){
    fts5ExprClearEof(pNode->apChild[i]);
  }
  pNode->bEof = 0;
}
void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
  fts5ExprClearEof(pExpr->pRoot);
}

/*
** This function is only called for detail=columns tables. 
*/
int sqlite3Fts5ExprPhraseCollist(
  Fts5Expr *pExpr, 
  int iPhrase, 
  const u8 **ppCollist, 








<
<
<
<
<
<
<
<
<
<
<








2613
2614
2615
2616
2617
2618
2619











2620
2621
2622
2623
2624
2625
2626

  return 1;
}

void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
  fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
}












/*
** This function is only called for detail=columns tables. 
*/
int sqlite3Fts5ExprPhraseCollist(
  Fts5Expr *pExpr, 
  int iPhrase, 
  const u8 **ppCollist,
︙︙
Changes to ext/fts5/fts5_main.c.
︙︙
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196

    assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
    assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
    assert( pCsr->iLastRowid==LARGEST_INT64 );
    assert( pCsr->iFirstRowid==SMALLEST_INT64 );
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
    sqlite3Fts5ExprClearEof(pCsr->pExpr);
  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
    if( zExpr==0 ) zExpr = "";

    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
    if( rc==SQLITE_OK ){
      if( zExpr[0]=='*' ){








<








1182
1183
1184
1185
1186
1187
1188

1189
1190
1191
1192
1193
1194
1195

    assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
    assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
    assert( pCsr->iLastRowid==LARGEST_INT64 );
    assert( pCsr->iFirstRowid==SMALLEST_INT64 );
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);

  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
    if( zExpr==0 ) zExpr = "";

    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
    if( rc==SQLITE_OK ){
      if( zExpr[0]=='*' ){
︙︙
Changes to ext/fts5/test/fts5rank.test.
︙︙
87
88
89
90
91
92
93


94

95


96
























97
98
99
100

} {1 3 2}

do_test 2.7 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
} {1 3 2}




































finish_test









>
>
|
>
|
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129

} {1 3 2}

do_test 2.7 {
  execsql { SELECT rowid FROM tt('a') ORDER BY rank; } db
} {1 3 2}


#--------------------------------------------------------------------------
# At one point there was a problem with queries such as:
#
#   ... MATCH 'x OR y' ORDER BY rank;
#
# if there were zero occurrences of token 'y' in the dataset. The
# following tests verify that that problem has been addressed.
#
foreach_detail_mode $::testprefix {
  do_execsql_test 3.0 {
    CREATE VIRTUAL TABLE y1 USING fts5(z, detail=%DETAIL%);
    INSERT INTO y1 VALUES('test xyz');
    INSERT INTO y1 VALUES('test test xyz test');
    INSERT INTO y1 VALUES('test test xyz');
  }

  do_execsql_test 3.1 {
    SELECT rowid FROM y1('test OR tset');
  } {1 2 3}

  do_execsql_test 3.2 {
    SELECT rowid FROM y1('test OR tset') ORDER BY bm25(y1)
  } {2 3 1}

  do_execsql_test 3.3 {
    SELECT rowid FROM y1('test OR tset') ORDER BY +rank
  } {2 3 1}

  do_execsql_test 3.4 {
    SELECT rowid FROM y1('test OR tset') ORDER BY rank
  } {2 3 1}
}


finish_test