SQLite

Check-in [49be646cd9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug preventing FTS from correctly processing bracket tokens that are immediately preceded by characters that are neither whitespace or token characters.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 49be646cd981f8ff0434cf90d2748afa30260017
User & Date: dan 2013-11-04 08:56:22.163
Context
2013-11-07
16:08
Add support for WITHOUT ROWID tables. This change also includes (1) standardization of the error message returned from run-time constraint errors, (2) improved EXPLAIN comments, (3) the SQLITE_ENABLE_EXPLAIN_COMMENTS option, (4) the SQLITE_ENABLE_MODULE_COMMENTS option, and (5) a bug fix (see [573cc27427]) in the handling of REPLACE on the rowid when secondary indices use FAIL or IGNORE. (check-in: c80e229dd9 user: drh tags: trunk)
2013-11-04
08:56
Fix a bug preventing FTS from correctly processing bracket tokens that are immediately preceded by characters that are neither whitespace or token characters. (check-in: 49be646cd9 user: dan tags: trunk)
2013-11-02
11:34
A pair of sqlite3_analyzer bug fixes: (1) quote strings in the SQL at the end of the output. (2) Fix test_stat.c so that it no longer misses some overflow pages on internal index pages. (check-in: 42a11e7464 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_expr.c.
151
152
153
154
155
156
157





158
159
160
161
162
163
164
      }
    }
  }
  *ppCsr = pCsr;
  return rc;
}







/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is
** reached before a token is found, set *ppExpr to zero. It is the







>
>
>
>
>







151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
      }
    }
  }
  *ppCsr = pCsr;
  return rc;
}

/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);

/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is
** reached before a token is found, set *ppExpr to zero. It is the
185
186
187
188
189
190
191








192
















193
194
195
196
197
198
199
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
    int nByte;                               /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);








    if( rc==SQLITE_OK ){
















      nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
      pRet = (Fts3Expr *)fts3MallocZero(nByte);
      if( !pRet ){
        rc = SQLITE_NOMEM;
      }else{
        pRet->eType = FTSQUERY_PHRASE;
        pRet->pPhrase = (Fts3Phrase *)&pRet[1];







>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
    int nByte;                               /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);

    if( (rc==SQLITE_OK || rc==SQLITE_DONE) && sqlite3_fts3_enable_parentheses ){
      int i;
      if( rc==SQLITE_DONE ) iStart = n;
      for(i=0; i<iStart; i++){
        if( z[i]=='(' ){
          pParse->nNest++;
          rc = fts3ExprParse(pParse, &z[i+1], n-i-1, &pRet, &nConsumed);
          if( rc==SQLITE_OK && !pRet ){
            rc = SQLITE_DONE;
          }
          nConsumed = (int)(i + 1 + nConsumed);
          break;
        }

        if( z[i]==')' ){
          rc = SQLITE_DONE;
          pParse->nNest--;
          nConsumed = i+1;
          break;
        }
      }
    }

    if( nConsumed==0 && rc==SQLITE_OK ){
      nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
      pRet = (Fts3Expr *)fts3MallocZero(nByte);
      if( !pRet ){
        rc = SQLITE_NOMEM;
      }else{
        pRet->eType = FTSQUERY_PHRASE;
        pRet->pPhrase = (Fts3Phrase *)&pRet[1];
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
  }
  sqlite3_free(zTemp);
  sqlite3_free(p);
  *ppExpr = 0;
  return SQLITE_NOMEM;
}

/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);

/*
** The output variable *ppExpr is populated with an allocated Fts3Expr 
** structure, or set to 0 if the end of the input buffer is reached.
**
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
** If SQLITE_ERROR is returned, pContext is populated with an error message.







<
<
<
<
<
<







394
395
396
397
398
399
400






401
402
403
404
405
406
407
  }
  sqlite3_free(zTemp);
  sqlite3_free(p);
  *ppExpr = 0;
  return SQLITE_NOMEM;
}







/*
** The output variable *ppExpr is populated with an allocated Fts3Expr 
** structure, or set to 0 if the end of the input buffer is reached.
**
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
** If SQLITE_ERROR is returned, pContext is populated with an error message.
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501

      /* Turns out that wasn't a keyword after all. This happens if the
      ** user has supplied a token such as "ORacle". Continue.
      */
    }
  }

  /* Check for an open bracket. */
  if( sqlite3_fts3_enable_parentheses ){
    if( *zInput=='(' ){
      int nConsumed;
      pParse->nNest++;
      rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
      if( rc==SQLITE_OK && !*ppExpr ){
        rc = SQLITE_DONE;
      }
      *pnConsumed = (int)((zInput - z) + 1 + nConsumed);
      return rc;
    }
  
    /* Check for a close bracket. */
    if( *zInput==')' ){
      pParse->nNest--;
      *pnConsumed = (int)((zInput - z) + 1);
      return SQLITE_DONE;
    }
  }

  /* See if we are dealing with a quoted phrase. If this is the case, then
  ** search for the closing quote and pass the whole string to getNextString()
  ** for processing. This is easy to do, as fts3 has no syntax for escaping
  ** a quote character embedded in a string.
  */
  if( *zInput=='"' ){
    for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







490
491
492
493
494
495
496





















497
498
499
500
501
502
503

      /* Turns out that wasn't a keyword after all. This happens if the
      ** user has supplied a token such as "ORacle". Continue.
      */
    }
  }






















  /* See if we are dealing with a quoted phrase. If this is the case, then
  ** search for the closing quote and pass the whole string to getNextString()
  ** for processing. This is easy to do, as fts3 has no syntax for escaping
  ** a quote character embedded in a string.
  */
  if( *zInput=='"' ){
    for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);
Changes to test/fts3expr.test.
24
25
26
27
28
29
30

31
32
33
34
35
36
37
}

set sqlite_fts3_enable_parentheses 1

proc test_fts3expr {expr} {
  db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
}

do_test fts3expr-1.0 {
  test_fts3expr "abcd"
} {PHRASE 3 0 abcd}
do_test fts3expr-1.1 {
  test_fts3expr " tag "
} {PHRASE 3 0 tag}








>







24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
}

set sqlite_fts3_enable_parentheses 1

proc test_fts3expr {expr} {
  db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
}

do_test fts3expr-1.0 {
  test_fts3expr "abcd"
} {PHRASE 3 0 abcd}
do_test fts3expr-1.1 {
  test_fts3expr " tag "
} {PHRASE 3 0 tag}

491
492
493
494
495
496
497












498
499
    CREATE VIRTUAL TABLE test USING fts3 (keyword);
    INSERT INTO test VALUES ('abc');
    SELECT * FROM test WHERE keyword MATCH '""';
  }
} {}














set sqlite_fts3_enable_parentheses 0
finish_test







>
>
>
>
>
>
>
>
>
>
>
>


492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
    CREATE VIRTUAL TABLE test USING fts3 (keyword);
    INSERT INTO test VALUES ('abc');
    SELECT * FROM test WHERE keyword MATCH '""';
  }
} {}


do_test fts3expr-8.0 { test_fts3expr "(blah)" } {PHRASE 3 0 blah}
do_test fts3expr-8.1 { test_fts3expr "(blah.)" } {PHRASE 3 0 blah}
do_test fts3expr-8.2 { test_fts3expr "(blah,)" } {PHRASE 3 0 blah}
do_test fts3expr-8.3 { test_fts3expr "(blah!)" } {PHRASE 3 0 blah}
do_test fts3expr-8.4 { test_fts3expr "(blah-)" } {PHRASE 3 0 blah}

do_test fts3expr-8.5 { test_fts3expr "((blah.))" } {PHRASE 3 0 blah}
do_test fts3expr-8.6 { test_fts3expr "(((blah,)))" } {PHRASE 3 0 blah}
do_test fts3expr-8.7 { test_fts3expr "((((blah!))))" } {PHRASE 3 0 blah}

do_test fts3expr-8.8 { test_fts3expr "(,(blah-),)" } {PHRASE 3 0 blah}

set sqlite_fts3_enable_parentheses 0
finish_test