Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
f1e942a1dda496a509741e9cc2a17e8b |
User & Date: | dan 2015-05-25 10:57:13.025 |
Context
2015-05-25
| ||
15:03 | Fix an assert() in btree routine freeSpace() that may be false if the database is corrupt. (check-in: 00a473c561 user: dan tags: trunk) | |
10:57 | Fix an fts4 matchinf() problem triggered by deferred tokens that are part of phrases that are part of AND expressions. (check-in: f1e942a1dd user: dan tags: trunk) | |
10:29 | Fix an obsolete comment - no changes to code. (check-in: 4e4228c47f user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
5070 5071 5072 5073 5074 5075 5076 | ** is advanced to the next row that contains an instance of "A * C", ** where "*" may match any single token. The position list in this case ** is populated as for "A * C" before returning. ** ** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is ** advanced to point to the next row that matches "x AND y". ** | | | 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 | ** is advanced to the next row that contains an instance of "A * C", ** where "*" may match any single token. The position list in this case ** is populated as for "A * C" before returning. ** ** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is ** advanced to point to the next row that matches "x AND y". ** ** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is ** really a match, taking into account deferred tokens and NEAR operators. */ static void fts3EvalNextRow( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pExpr, /* Expr. to advance to next matching row */ int *pRc /* IN/OUT: Error code */ ){ |
︙ | ︙ | |||
5290 5291 5292 5293 5294 5295 5296 | } } return res; } /* | | | 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 | } } return res; } /* ** This function is a helper function for sqlite3Fts3EvalTestDeferred(). ** Assuming no error occurs or has occurred, It returns non-zero if the ** expression passed as the second argument matches the row that pCsr ** currently points to, or zero if it does not. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** If an error occurs during execution of this function, *pRc is set to ** the appropriate SQLite error code. In this case the returned value is |
︙ | ︙ | |||
5411 5412 5413 5414 5415 5416 5417 | ** ** 2. After scanning the current FTS table row for the deferred tokens, ** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ | | | 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 | ** ** 2. After scanning the current FTS table row for the deferred tokens, ** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ /* If there are one or more deferred tokens, load the current row into ** memory and scan it to determine the position list for each deferred ** token. Then, see if this row is really a match, considering deferred |
︙ | ︙ | |||
5458 5459 5460 5461 5462 5463 5464 | } assert( sqlite3_data_count(pCsr->pStmt)==0 ); fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; | | | 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 | } assert( sqlite3_data_count(pCsr->pStmt)==0 ); fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); } /* Check if the cursor is past the end of the docid range specified ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ if( rc==SQLITE_OK && ( (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) || (pCsr->bDesc!=0 && pCsr->iPrevId<pCsr->iMinDocid) |
︙ | ︙ | |||
5619 5620 5621 5622 5623 5624 5625 | fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR | | | 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 | fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ fts3EvalUpdateCounts(pRoot); } } |
︙ | ︙ | |||
5644 5645 5646 5647 5648 5649 5650 | ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); */ fts3EvalRestart(pCsr, pRoot, &rc); do { fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); | < | 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 | ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); */ fts3EvalRestart(pCsr, pRoot, &rc); do { fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); } } return rc; } /* ** This function is used by the matchinfo() module to query a phrase |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
547 548 549 550 551 552 553 554 555 556 557 558 559 560 | int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); void sqlite3Fts3CreateStatTable(int*, Fts3Table*); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, sqlite3_tokenizer **, char ** ); | > | 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 | int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); void sqlite3Fts3CreateStatTable(int*, Fts3Table*); int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, sqlite3_tokenizer **, char ** ); |
︙ | ︙ |
Changes to ext/fts3/fts3_snippet.c.
︙ | ︙ | |||
1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 | if( rc!=SQLITE_OK ) break; if( bGlobal ){ if( pCsr->pDeferred ){ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); if( rc!=SQLITE_OK ) break; } rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); if( rc!=SQLITE_OK ) break; } (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); break; } } | > | 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 | if( rc!=SQLITE_OK ) break; if( bGlobal ){ if( pCsr->pDeferred ){ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); if( rc!=SQLITE_OK ) break; } rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); sqlite3Fts3EvalTestDeferred(pCsr, &rc); if( rc!=SQLITE_OK ) break; } (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); break; } } |
︙ | ︙ |
Changes to test/fts3defer2.test.
︙ | ︙ | |||
117 118 119 120 121 122 123 124 125 126 127 128 129 130 | do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ [list 1 2 2 1 2 2 0 54 54] \ ] set sqlite_fts3_enable_parentheses 0 } do_execsql_test 2.3.1 { CREATE VIRTUAL TABLE t3 USING fts4; INSERT INTO t3 VALUES('a b c d e f'); INSERT INTO t3 VALUES('x b c d e f'); INSERT INTO t3 VALUES('d e f a b c'); | > > > > > > | 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ [list 1 2 2 1 2 2 0 54 54] \ ] set sqlite_fts3_enable_parentheses 0 do_execsql_test 2.2.$tn.4 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'e "g z"'; } [list \ [list 1 2 2 1 2 2] \ ] } do_execsql_test 2.3.1 { CREATE VIRTUAL TABLE t3 USING fts4; INSERT INTO t3 VALUES('a b c d e f'); INSERT INTO t3 VALUES('x b c d e f'); INSERT INTO t3 VALUES('d e f a b c'); |
︙ | ︙ |