/ Check-in [49be646c]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a bug preventing FTS from correctly processing bracket tokens that are immediately preceded by characters that are neither whitespace or token characters.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:49be646cd981f8ff0434cf90d2748afa30260017
User & Date: dan 2013-11-04 08:56:22
Context
2013-11-07
16:08
Add support for WITHOUT ROWID tables. This change also includes (1) standardization of the error message returned from run-time constraint errors, (2) improved EXPLAIN comments, (3) the SQLITE_ENABLE_EXPLAIN_COMMENTS option, (4) the SQLITE_ENABLE_MODULE_COMMENTS option, and (5) a bug fix (see [573cc27427]) in the handling of REPLACE on the rowid when secondary indices use FAIL or IGNORE. check-in: c80e229d user: drh tags: trunk
2013-11-04
08:56
Fix a bug preventing FTS from correctly processing bracket tokens that are immediately preceded by characters that are neither whitespace or token characters. check-in: 49be646c user: dan tags: trunk
2013-11-02
11:34
A pair of sqlite3_analyzer bug fixes: (1) quote strings in the SQL at the end of the output. (2) Fix test_stat.c so that it no longer misses some overflow pages on internal index pages. check-in: 42a11e74 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_expr.c.

151
152
153
154
155
156
157





158
159
160
161
162
163
164
...
185
186
187
188
189
190
191








192
















193
194
195
196
197
198
199
...
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
...
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
      }
    }
  }
  *ppCsr = pCsr;
  return rc;
}







/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is
** reached before a token is found, set *ppExpr to zero. It is the
................................................................................
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
    int nByte;                               /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);








    if( rc==SQLITE_OK ){
















      nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
      pRet = (Fts3Expr *)fts3MallocZero(nByte);
      if( !pRet ){
        rc = SQLITE_NOMEM;
      }else{
        pRet->eType = FTSQUERY_PHRASE;
        pRet->pPhrase = (Fts3Phrase *)&pRet[1];
................................................................................
  }
  sqlite3_free(zTemp);
  sqlite3_free(p);
  *ppExpr = 0;
  return SQLITE_NOMEM;
}

/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);

/*
** The output variable *ppExpr is populated with an allocated Fts3Expr 
** structure, or set to 0 if the end of the input buffer is reached.
**
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
** If SQLITE_ERROR is returned, pContext is populated with an error message.
................................................................................

      /* Turns out that wasn't a keyword after all. This happens if the
      ** user has supplied a token such as "ORacle". Continue.
      */
    }
  }

  /* Check for an open bracket. */
  if( sqlite3_fts3_enable_parentheses ){
    if( *zInput=='(' ){
      int nConsumed;
      pParse->nNest++;
      rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed);
      if( rc==SQLITE_OK && !*ppExpr ){
        rc = SQLITE_DONE;
      }
      *pnConsumed = (int)((zInput - z) + 1 + nConsumed);
      return rc;
    }
  
    /* Check for a close bracket. */
    if( *zInput==')' ){
      pParse->nNest--;
      *pnConsumed = (int)((zInput - z) + 1);
      return SQLITE_DONE;
    }
  }

  /* See if we are dealing with a quoted phrase. If this is the case, then
  ** search for the closing quote and pass the whole string to getNextString()
  ** for processing. This is easy to do, as fts3 has no syntax for escaping
  ** a quote character embedded in a string.
  */
  if( *zInput=='"' ){
    for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);







>
>
>
>
>







 







>
>
>
>
>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<
<
<
<
<
<







 







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
...
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
...
394
395
396
397
398
399
400






401
402
403
404
405
406
407
...
490
491
492
493
494
495
496





















497
498
499
500
501
502
503
      }
    }
  }
  *ppCsr = pCsr;
  return rc;
}

/*
** Function getNextNode(), which is called by fts3ExprParse(), may itself
** call fts3ExprParse(). So this forward declaration is required.
*/
static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *);

/*
** Extract the next token from buffer z (length n) using the tokenizer
** and other information (column names etc.) in pParse. Create an Fts3Expr
** structure of type FTSQUERY_PHRASE containing a phrase consisting of this
** single token and set *ppExpr to point to it. If the end of the buffer is
** reached before a token is found, set *ppExpr to zero. It is the
................................................................................
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, n, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
    int nByte;                               /* total space to allocate */

    rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition);

    if( (rc==SQLITE_OK || rc==SQLITE_DONE) && sqlite3_fts3_enable_parentheses ){
      int i;
      if( rc==SQLITE_DONE ) iStart = n;
      for(i=0; i<iStart; i++){
        if( z[i]=='(' ){
          pParse->nNest++;
          rc = fts3ExprParse(pParse, &z[i+1], n-i-1, &pRet, &nConsumed);
          if( rc==SQLITE_OK && !pRet ){
            rc = SQLITE_DONE;
          }
          nConsumed = (int)(i + 1 + nConsumed);
          break;
        }

        if( z[i]==')' ){
          rc = SQLITE_DONE;
          pParse->nNest--;
          nConsumed = i+1;
          break;
        }
      }
    }

    if( nConsumed==0 && rc==SQLITE_OK ){
      nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken;
      pRet = (Fts3Expr *)fts3MallocZero(nByte);
      if( !pRet ){
        rc = SQLITE_NOMEM;
      }else{
        pRet->eType = FTSQUERY_PHRASE;
        pRet->pPhrase = (Fts3Phrase *)&pRet[1];
................................................................................
  }
  sqlite3_free(zTemp);
  sqlite3_free(p);
  *ppExpr = 0;
  return SQLITE_NOMEM;
}







/*
** The output variable *ppExpr is populated with an allocated Fts3Expr 
** structure, or set to 0 if the end of the input buffer is reached.
**
** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM
** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered.
** If SQLITE_ERROR is returned, pContext is populated with an error message.
................................................................................

      /* Turns out that wasn't a keyword after all. This happens if the
      ** user has supplied a token such as "ORacle". Continue.
      */
    }
  }






















  /* See if we are dealing with a quoted phrase. If this is the case, then
  ** search for the closing quote and pass the whole string to getNextString()
  ** for processing. This is easy to do, as fts3 has no syntax for escaping
  ** a quote character embedded in a string.
  */
  if( *zInput=='"' ){
    for(ii=1; ii<nInput && zInput[ii]!='"'; ii++);

Changes to test/fts3expr.test.

24
25
26
27
28
29
30

31
32
33
34
35
36
37
...
491
492
493
494
495
496
497












498
499
}

set sqlite_fts3_enable_parentheses 1

proc test_fts3expr {expr} {
  db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
}

do_test fts3expr-1.0 {
  test_fts3expr "abcd"
} {PHRASE 3 0 abcd}
do_test fts3expr-1.1 {
  test_fts3expr " tag "
} {PHRASE 3 0 tag}

................................................................................
    CREATE VIRTUAL TABLE test USING fts3 (keyword);
    INSERT INTO test VALUES ('abc');
    SELECT * FROM test WHERE keyword MATCH '""';
  }
} {}














set sqlite_fts3_enable_parentheses 0
finish_test







>







 







>
>
>
>
>
>
>
>
>
>
>
>


24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
...
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
}

set sqlite_fts3_enable_parentheses 1

proc test_fts3expr {expr} {
  db one {SELECT fts3_exprtest('simple', $expr, 'a', 'b', 'c')}
}

do_test fts3expr-1.0 {
  test_fts3expr "abcd"
} {PHRASE 3 0 abcd}
do_test fts3expr-1.1 {
  test_fts3expr " tag "
} {PHRASE 3 0 tag}

................................................................................
    CREATE VIRTUAL TABLE test USING fts3 (keyword);
    INSERT INTO test VALUES ('abc');
    SELECT * FROM test WHERE keyword MATCH '""';
  }
} {}


do_test fts3expr-8.0 { test_fts3expr "(blah)" } {PHRASE 3 0 blah}
do_test fts3expr-8.1 { test_fts3expr "(blah.)" } {PHRASE 3 0 blah}
do_test fts3expr-8.2 { test_fts3expr "(blah,)" } {PHRASE 3 0 blah}
do_test fts3expr-8.3 { test_fts3expr "(blah!)" } {PHRASE 3 0 blah}
do_test fts3expr-8.4 { test_fts3expr "(blah-)" } {PHRASE 3 0 blah}

do_test fts3expr-8.5 { test_fts3expr "((blah.))" } {PHRASE 3 0 blah}
do_test fts3expr-8.6 { test_fts3expr "(((blah,)))" } {PHRASE 3 0 blah}
do_test fts3expr-8.7 { test_fts3expr "((((blah!))))" } {PHRASE 3 0 blah}

do_test fts3expr-8.8 { test_fts3expr "(,(blah-),)" } {PHRASE 3 0 blah}

set sqlite_fts3_enable_parentheses 0
finish_test