SQLite

Check-in [df36ac9481]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-content
Files: files | file ages | folders
SHA1: df36ac948179f37b432a88701b6c79299e073ce8
User & Date: dan 2011-10-19 15:52:48.921
Context
2011-10-19
16:20
Merge the fts4-content branch with the trunk. (check-in: 8a4077057d user: dan tags: trunk)
15:52
Have FTS3 ignore ^ prefixes. The ^ syntax is only supported on FTS4 tables. (Closed-Leaf check-in: df36ac9481 user: dan tags: fts4-content)
11:57
Change the way ^ tokens work in FTS so that the filtering is done as part of reading the FTS index instead of waiting until an entire doclist has been retrieved and then filtering it. (check-in: 9b58c59eb4 user: dan tags: fts4-content)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3.c.
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
    int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
    const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);

    if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
      return SQLITE_NOMEM;
    }

    rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, 
        iCol, zQuery, -1, &pCsr->pExpr
    );
    if( rc!=SQLITE_OK ){
      if( rc==SQLITE_ERROR ){
        static const char *zErr = "malformed MATCH expression: [%s]";
        p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
      }
      return rc;







|
|







2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
    int iCol = idxNum-FTS3_FULLTEXT_SEARCH;
    const char *zQuery = (const char *)sqlite3_value_text(apVal[0]);

    if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
      return SQLITE_NOMEM;
    }

    rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->bHasStat, 
        p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
    );
    if( rc!=SQLITE_OK ){
      if( rc==SQLITE_ERROR ){
        static const char *zErr = "malformed MATCH expression: [%s]";
        p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
      }
      return rc;
Changes to ext/fts3/fts3Int.h.
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
  const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif








|







485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
  const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
int sqlite3Fts3InitTerm(sqlite3 *db);
#endif

Changes to ext/fts3/fts3_expr.c.
89
90
91
92
93
94
95

96
97
98
99
100
101
102
**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
**   zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
  sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
  const char **azCol;                 /* Array of column names for fts3 table */

  int nCol;                           /* Number of entries in azCol[] */
  int iDefaultCol;                    /* Default column to query */
  int isNot;                          /* True if getNextNode() sees a unary - */
  sqlite3_context *pCtx;              /* Write error message here */
  int nNest;                          /* Number of nested brackets */
};








>







89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
**   FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
**   zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
  sqlite3_tokenizer *pTokenizer;      /* Tokenizer module */
  const char **azCol;                 /* Array of column names for fts3 table */
  int bFts4;                          /* True to allow FTS4-only syntax */
  int nCol;                           /* Number of entries in azCol[] */
  int iDefaultCol;                    /* Default column to query */
  int isNot;                          /* True if getNextNode() sees a unary - */
  sqlite3_context *pCtx;              /* Write error message here */
  int nNest;                          /* Number of nested brackets */
};

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197

        while( 1 ){
          if( !sqlite3_fts3_enable_parentheses 
           && iStart>0 && z[iStart-1]=='-' 
          ){
            pParse->isNot = 1;
            iStart--;
          }else if( iStart>0 && z[iStart-1]=='^' ){
            pRet->pPhrase->aToken[0].bFirst = 1;
            iStart--;
          }else{
            break;
          }
        }








|







184
185
186
187
188
189
190
191
192
193
194
195
196
197
198

        while( 1 ){
          if( !sqlite3_fts3_enable_parentheses 
           && iStart>0 && z[iStart-1]=='-' 
          ){
            pParse->isNot = 1;
            iStart--;
          }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){
            pRet->pPhrase->aToken[0].bFirst = 1;
            iStart--;
          }else{
            break;
          }
        }

737
738
739
740
741
742
743

744
745
746
747
748
749
750
751
752
753
754
755
756

757
758
759
760
761
762
763
** column to match against for tokens for which a column name is not explicitly
** specified as part of the query string), or -1 if tokens may by default
** match any table column.
*/
int sqlite3Fts3ExprParse(
  sqlite3_tokenizer *pTokenizer,      /* Tokenizer module */
  char **azCol,                       /* Array of column names for fts3 table */

  int nCol,                           /* Number of entries in azCol[] */
  int iDefaultCol,                    /* Default column to query */
  const char *z, int n,               /* Text of MATCH query */
  Fts3Expr **ppExpr                   /* OUT: Parsed query structure */
){
  int nParsed;
  int rc;
  ParseContext sParse;
  sParse.pTokenizer = pTokenizer;
  sParse.azCol = (const char **)azCol;
  sParse.nCol = nCol;
  sParse.iDefaultCol = iDefaultCol;
  sParse.nNest = 0;

  if( z==0 ){
    *ppExpr = 0;
    return SQLITE_OK;
  }
  if( n<0 ){
    n = (int)strlen(z);
  }







>













>







738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
** column to match against for tokens for which a column name is not explicitly
** specified as part of the query string), or -1 if tokens may by default
** match any table column.
*/
int sqlite3Fts3ExprParse(
  sqlite3_tokenizer *pTokenizer,      /* Tokenizer module */
  char **azCol,                       /* Array of column names for fts3 table */
  int bFts4,                          /* True to allow FTS4-only syntax */
  int nCol,                           /* Number of entries in azCol[] */
  int iDefaultCol,                    /* Default column to query */
  const char *z, int n,               /* Text of MATCH query */
  Fts3Expr **ppExpr                   /* OUT: Parsed query structure */
){
  int nParsed;
  int rc;
  ParseContext sParse;
  sParse.pTokenizer = pTokenizer;
  sParse.azCol = (const char **)azCol;
  sParse.nCol = nCol;
  sParse.iDefaultCol = iDefaultCol;
  sParse.nNest = 0;
  sParse.bFts4 = bFts4;
  if( z==0 ){
    *ppExpr = 0;
    return SQLITE_OK;
  }
  if( n<0 ){
    n = (int)strlen(z);
  }
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
    goto exprtest_out;
  }
  for(ii=0; ii<nCol; ii++){
    azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  }

  rc = sqlite3Fts3ExprParse(
      pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr
  );
  if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
    sqlite3_result_error(context, "Error parsing expression", -1);
  }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
    sqlite3_result_error_nomem(context);
  }else{
    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);







|







942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
    goto exprtest_out;
  }
  for(ii=0; ii<nCol; ii++){
    azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]);
  }

  rc = sqlite3Fts3ExprParse(
      pTokenizer, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr
  );
  if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){
    sqlite3_result_error(context, "Error parsing expression", -1);
  }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){
    sqlite3_result_error_nomem(context);
  }else{
    sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
Changes to test/fts3defer.test.
423
424
425
426
427
428
429




430
431
432
433
434
435

436
437
438
439
440
441
442
  } {15 26 92 96}
  if {$fts3_simple_deferred_tokens_only==0} {
    do_select_test 6.2.3 {
      SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
    } {8 15 26 92 96}
  }





  do_select_test 7.1 {
    SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
  } {56 62}
  do_select_test 7.2 {
    SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
  } {43}

}

set testprefix fts3defer

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE x1 USING fts4(a, b);
  INSERT INTO x1 VALUES('a b c', 'd e f');







>
>
>
>
|
|
|
|
|
|
>







423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
  } {15 26 92 96}
  if {$fts3_simple_deferred_tokens_only==0} {
    do_select_test 6.2.3 {
      SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
    } {8 15 26 92 96}
  }

  if {$tn>1} {
    # These tests will not work with $tn==1, as in this case table t1 is
    # created using FTS3. The ^ syntax is only available with FTS4 tables.
    #
    do_select_test 7.1 {
      SELECT rowid FROM t1 WHERE t1 MATCH '^zm mjpavjuhw'
    } {56 62}
    do_select_test 7.2 {
      SELECT rowid FROM t1 WHERE t1 MATCH '^azavwm zm'
    } {43}
  }
}

set testprefix fts3defer

do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE x1 USING fts4(a, b);
  INSERT INTO x1 VALUES('a b c', 'd e f');
Changes to test/fts3first.test.
14
15
16
17
18
19
20


21
22
23
24
25
26
27
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}



proc lreverse {L} {
  set res [list]
  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
    lappend res [lindex $L $ii]
  }
  set res
}







>
>







14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
source $testdir/malloc_common.tcl

ifcapable !fts3 {
  finish_test
  return
}

set testprefix fts3first

proc lreverse {L} {
  set res [list]
  for {set ii [expr [llength $L]-1]} {$ii>=0} {incr ii -1} {
    lappend res [lindex $L $ii]
  }
  set res
}
134
135
136
137
138
139
140
141



















142
  do_execsql_test 1.3.$tn.1 {
    SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
  } $res
  do_execsql_test 1.3.$tn.2 {
    SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
  } $rev
}




















finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
  do_execsql_test 1.3.$tn.1 {
    SELECT mit(matchinfo(x1, 'x')) FROM x1 WHERE x1 MATCH $match
  } $res
  do_execsql_test 1.3.$tn.2 {
    SELECT mit(matchinfo(x2, 'x')) FROM x2 WHERE x2 MATCH $match
  } $rev
}

# Test that ^ is ignored for FTS3 tables.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE x3 USING fts3;
  INSERT INTO x3 VALUES('A B C');
  INSERT INTO x3 VALUES('B A C');

  CREATE VIRTUAL TABLE x4 USING fts4;
  INSERT INTO x4 VALUES('A B C');
  INSERT INTO x4 VALUES('B A C');
}

do_execsql_test 2.2.1 {
  SELECT * FROM x3 WHERE x3 MATCH '^A';
} {{A B C} {B A C}}
do_execsql_test 2.2.2 {
  SELECT * FROM x4 WHERE x4 MATCH '^A';
} {{A B C}}

finish_test