Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts4-content |
| Files: | files | file ages | folders |
| SHA1: |
df36ac948179f37b432a88701b6c7929 |
| User & Date: | dan 2011-10-19 15:52:48 |
Context
|
2011-10-19
| ||
| 16:20 | Merge the fts4-content branch with the trunk. (check-in: 8a407705 user: dan tags: trunk) | |
| 15:52 | Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. (Closed-Leaf check-in: df36ac94 user: dan tags: fts4-content) | |
| 11:57 | Change the way ^ tokens work in FTS so that the filtering is done as part of reading the FTS index instead of waiting until an entire doclist has been retrieved and then filtering it. (check-in: 9b58c59e user: dan tags: fts4-content) | |
Changes
Changes to ext/fts3/fts3.c.
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 |
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
return SQLITE_NOMEM;
}
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn,
iCol, zQuery, -1, &pCsr->pExpr
);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_ERROR ){
static const char *zErr = "malformed MATCH expression: [%s]";
p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
}
return rc;
|
| | |
2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 |
int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);
if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
return SQLITE_NOMEM;
}
rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat,
p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
);
if( rc!=SQLITE_OK ){
if( rc==SQLITE_ERROR ){
static const char *zErr = "malformed MATCH expression: [%s]";
p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
}
return rc;
|
Changes to ext/fts3/fts3Int.h.
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 |
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); int sqlite3Fts3InitTerm(sqlite3 *db); #endif |
| |
485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 |
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *,
char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif
|
Changes to ext/fts3/fts3_expr.c.
89 90 91 92 93 94 95 96 97 98 99 100 101 102 ... 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 ... 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 ... 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 |
** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
** zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
const char **azCol; /* Array of column names for fts3 table */
int nCol; /* Number of entries in azCol[] */
int iDefaultCol; /* Default column to query */
int isNot; /* True if getNextNode() sees a unary - */
sqlite3_context *pCtx; /* Write error message here */
int nNest; /* Number of nested brackets */
};
................................................................................
while( 1 ){
if( !sqlite3_fts3_enable_parentheses
&& iStart>0 && z[iStart-1]=='-'
){
pParse->isNot = 1;
iStart--;
}else if( iStart>0 && z[iStart-1]=='^' ){
pRet->pPhrase->aToken[0].bFirst = 1;
iStart--;
}else{
break;
}
}
................................................................................
** column to match against for tokens for which a column name is not explicitly
** specified as part of the query string), or -1 if tokens may by default
** match any table column.
*/
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
char **azCol, /* Array of column names for fts3 table */
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr /* OUT: Parsed query structure */
){
int nParsed;
int rc;
ParseContext sParse;
sParse.pTokenizer = pTokenizer;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.nNest = 0;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
................................................................................
goto exprtest_out;
}
for(ii=0; ii<nCol; ii++){
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
}else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
sqlite3_result_error_nomem(context);
}else{
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
|
> | > > | |
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 ... 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 ... 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 ... 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 |
** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
** zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
const char **azCol; /* Array of column names for fts3 table */
int bFts4; /* True to allow FTS4-only syntax */
int nCol; /* Number of entries in azCol[] */
int iDefaultCol; /* Default column to query */
int isNot; /* True if getNextNode() sees a unary - */
sqlite3_context *pCtx; /* Write error message here */
int nNest; /* Number of nested brackets */
};
................................................................................
while( 1 ){
if( !sqlite3_fts3_enable_parentheses
&& iStart>0 && z[iStart-1]=='-'
){
pParse->isNot = 1;
iStart--;
}else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){
pRet->pPhrase->aToken[0].bFirst = 1;
iStart--;
}else{
break;
}
}
................................................................................
** column to match against for tokens for which a column name is not explicitly
** specified as part of the query string), or -1 if tokens may by default
** match any table column.
*/
int sqlite3Fts3ExprParse(
sqlite3_tokenizer *pTokenizer, /* Tokenizer module */
char **azCol, /* Array of column names for fts3 table */
int bFts4, /* True to allow FTS4-only syntax */
int nCol, /* Number of entries in azCol[] */
int iDefaultCol, /* Default column to query */
const char *z, int n, /* Text of MATCH query */
Fts3Expr **ppExpr /* OUT: Parsed query structure */
){
int nParsed;
int rc;
ParseContext sParse;
sParse.pTokenizer = pTokenizer;
sParse.azCol = (const char **)azCol;
sParse.nCol = nCol;
sParse.iDefaultCol = iDefaultCol;
sParse.nNest = 0;
sParse.bFts4 = bFts4;
if( z==0 ){
*ppExpr = 0;
return SQLITE_OK;
}
if( n<0 ){
n = (int)strlen(z);
}
................................................................................
goto exprtest_out;
}
for(ii=0; ii<nCol; ii++){
azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
}
rc = sqlite3Fts3ExprParse(
pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
);
if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
sqlite3_result_error(context, "Error parsing expression", -1);
}else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
sqlite3_result_error_nomem(context);
}else{
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
|
Changes to test/fts3defer.test.
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 |
} {15 26 92 96}
if {$fts3_simple_deferred_tokens_only==0} {
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
do_select_test 7.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
} {56 62}
do_select_test 7.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
} {43}
}
set testprefix fts3defer
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE x1 USING fts4(a, b);
INSERT INTO x1 VALUES('a b c', 'd e f');
|
> > > > | | | | | | > |
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 |
} {15 26 92 96}
if {$fts3_simple_deferred_tokens_only==0} {
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
if {$tn>1} {
# These tests will not work with $tn==1, as in this case table t1 is
# created using FTS3. The ^ syntax is only available with FTS4 tables.
#
do_select_test 7.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
} {56 62}
do_select_test 7.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
} {43}
}
}
set testprefix fts3defer
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE x1 USING fts4(a, b);
INSERT INTO x1 VALUES('a b c', 'd e f');
|
Changes to test/fts3first.test.
14
15
16
17
18
19
20
21
22
23
24
25
26
27
...
134
135
136
137
138
139
140
141
142
|
source $testdir/malloc_common.tcl
ifcapable !fts3 {
finish_test
return
}
proc lreverse {L} {
set res [list]
for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
lappend res [lindex $L $ii]
}
set res
}
................................................................................
do_execsql_test 1.3.$tn.1 {
SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
} $res
do_execsql_test 1.3.$tn.2 {
SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
} $rev
}
finish_test
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
...
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
source $testdir/malloc_common.tcl
ifcapable !fts3 {
finish_test
return
}
set testprefix fts3first
proc lreverse {L} {
set res [list]
for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
lappend res [lindex $L $ii]
}
set res
}
................................................................................
do_execsql_test 1.3.$tn.1 {
SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
} $res
do_execsql_test 1.3.$tn.2 {
SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
} $rev
}
# Test that ^ is ignored for FTS3 tables.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE x3 USING fts3;
INSERT INTO x3 VALUES('A B C');
INSERT INTO x3 VALUES('B A C');
CREATE VIRTUAL TABLE x4 USING fts4;
INSERT INTO x4 VALUES('A B C');
INSERT INTO x4 VALUES('B A C');
}
do_execsql_test 2.2.1 {
SELECT * FROM x3 WHERE x3 MATCH '^A';
} {{A B C} {B A C}}
do_execsql_test 2.2.2 {
SELECT * FROM x4 WHERE x4 MATCH '^A';
} {{A B C}}
finish_test
|