Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-content |
Files: | files | file ages | folders |
SHA1: |
df36ac948179f37b432a88701b6c7929 |
User & Date: | dan 2011-10-19 15:52:48.921 |
Context
2011-10-19
| ||
16:20 | Merge the fts4-content branch with the trunk. (check-in: 8a4077057d user: dan tags: trunk) | |
15:52 | Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. (Closed-Leaf check-in: df36ac9481 user: dan tags: fts4-content) | |
11:57 | Change the way ^ tokens work in FTS so that the filtering is done as part of reading the FTS index instead of waiting until an entire doclist has been retrieved and then filtering it. (check-in: 9b58c59eb4 user: dan tags: fts4-content) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
2890 2891 2892 2893 2894 2895 2896 | int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ return SQLITE_NOMEM; } | | | | 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 | int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ return SQLITE_NOMEM; } rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ static const char *zErr = "malformed MATCH expression: [%s]"; p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
485 486 487 488 489 490 491 | void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, | | | 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 | void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); int sqlite3Fts3InitTerm(sqlite3 *db); #endif |
︙ | ︙ |
Changes to ext/fts3/fts3_expr.c.
︙ | ︙ | |||
89 90 91 92 93 94 95 96 97 98 99 100 101 102 | ** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to ** zero. */ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; | > | 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 | ** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to ** zero. */ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int bFts4; /* True to allow FTS4-only syntax */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; |
︙ | ︙ | |||
183 184 185 186 187 188 189 | while( 1 ){ if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ pParse->isNot = 1; iStart--; | | | 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | while( 1 ){ if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ pParse->isNot = 1; iStart--; }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ pRet->pPhrase->aToken[0].bFirst = 1; iStart--; }else{ break; } } |
︙ | ︙ | |||
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 | ** column to match against for tokens for which a column name is not explicitly ** specified as part of the query string), or -1 if tokens may by default ** match any table column. */ int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ char **azCol, /* Array of column names for fts3 table */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ int nParsed; int rc; ParseContext sParse; sParse.pTokenizer = pTokenizer; sParse.azCol = (const char **)azCol; sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; sParse.nNest = 0; if( z==0 ){ *ppExpr = 0; return SQLITE_OK; } if( n<0 ){ n = (int)strlen(z); } | > > | 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 | ** column to match against for tokens for which a column name is not explicitly ** specified as part of the query string), or -1 if tokens may by default ** match any table column. */ int sqlite3Fts3ExprParse( sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ char **azCol, /* Array of column names for fts3 table */ int bFts4, /* True to allow FTS4-only syntax */ int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ int nParsed; int rc; ParseContext sParse; sParse.pTokenizer = pTokenizer; sParse.azCol = (const char **)azCol; sParse.nCol = nCol; sParse.iDefaultCol = iDefaultCol; sParse.nNest = 0; sParse.bFts4 = bFts4; if( z==0 ){ *ppExpr = 0; return SQLITE_OK; } if( n<0 ){ n = (int)strlen(z); } |
︙ | ︙ | |||
939 940 941 942 943 944 945 | goto exprtest_out; } for(ii=0; ii<nCol; ii++){ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } rc = sqlite3Fts3ExprParse( | | | 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 | goto exprtest_out; } for(ii=0; ii<nCol; ii++){ azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } rc = sqlite3Fts3ExprParse( pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr ); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); }else{ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); |
︙ | ︙ |
Changes to test/fts3defer.test.
︙ | ︙ | |||
423 424 425 426 427 428 429 | } {15 26 92 96} if {$fts3_simple_deferred_tokens_only==0} { do_select_test 6.2.3 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } | > > > > | | | | | | > | 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 | } {15 26 92 96} if {$fts3_simple_deferred_tokens_only==0} { do_select_test 6.2.3 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' } {8 15 26 92 96} } if {$tn>1} { # These tests will not work with $tn==1, as in this case table t1 is # created using FTS3. The ^ syntax is only available with FTS4 tables. # do_select_test 7.1 { SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw' } {56 62} do_select_test 7.2 { SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm' } {43} } } set testprefix fts3defer do_execsql_test 3.1 { CREATE VIRTUAL TABLE x1 USING fts4(a, b); INSERT INTO x1 VALUES('a b c', 'd e f'); |
︙ | ︙ |
Changes to test/fts3first.test.
︙ | ︙ | |||
14 15 16 17 18 19 20 21 22 23 24 25 26 27 | source $testdir/malloc_common.tcl ifcapable !fts3 { finish_test return } proc lreverse {L} { set res [list] for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} { lappend res [lindex $L $ii] } set res } | > > | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | source $testdir/malloc_common.tcl ifcapable !fts3 { finish_test return } set testprefix fts3first proc lreverse {L} { set res [list] for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} { lappend res [lindex $L $ii] } set res } |
︙ | ︙ | |||
134 135 136 137 138 139 140 141 142 | do_execsql_test 1.3.$tn.1 { SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match } $res do_execsql_test 1.3.$tn.2 { SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match } $rev } finish_test | > > > > > > > > > > > > > > > > > > > | 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | do_execsql_test 1.3.$tn.1 { SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match } $res do_execsql_test 1.3.$tn.2 { SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match } $rev } # Test that ^ is ignored for FTS3 tables. # do_execsql_test 2.1 { CREATE VIRTUAL TABLE x3 USING fts3; INSERT INTO x3 VALUES('A B C'); INSERT INTO x3 VALUES('B A C'); CREATE VIRTUAL TABLE x4 USING fts4; INSERT INTO x4 VALUES('A B C'); INSERT INTO x4 VALUES('B A C'); } do_execsql_test 2.2.1 { SELECT * FROM x3 WHERE x3 MATCH '^A'; } {{A B C} {B A C}} do_execsql_test 2.2.2 { SELECT * FROM x4 WHERE x4 MATCH '^A'; } {{A B C}} finish_test |