/ Check-in [d4331943]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further improvements to test coverage of fts5 code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1:d4331943dff259380c4025bb740d8aba6972d351
User & Date: dan 2015-05-01 20:38:57
Context
2015-05-02
20:35
Reorganize some of the fts5 expression parsing code. Improve test coverage of the same. check-in: c4456dc5 user: dan tags: fts5
2015-05-01
20:38
Further improvements to test coverage of fts5 code. check-in: d4331943 user: dan tags: fts5
12:14
Improve test coverage of fts5.c. check-in: add4f468 user: dan tags: fts5
Changes
Hide Diffs Unified Diffs Show Whitespace Changes Patch

Changes to ext/fts5/fts5_expr.c.

100
101
102
103
104
105
106
107
108
109

110
111
112
113

114
115
116
117
118
119
120
...
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280

281
282
283
284
285






286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
...
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
...
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
...
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
  int rc;
  int nPhrase;                    /* Size of apPhrase array */
  Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
  Fts5ExprNode *pExpr;            /* Result of a successful parse */
};

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
  if( pParse->rc==SQLITE_OK ){
    va_list ap;
    va_start(ap, zFmt);

    pParse->zErr = sqlite3_vmprintf(zFmt, ap);
    va_end(ap);
    pParse->rc = SQLITE_ERROR;
  }

}

static int fts5ExprIsspace(char t){
  return t==' ' || t=='\t' || t=='\n' || t=='\r';
}

static int fts5ExprIstoken(char t){
................................................................................
int sqlite3Fts5ExprPhraseExpr(
  Fts5Config *pConfig,
  Fts5Expr *pExpr, 
  int iPhrase, 
  Fts5Expr **ppNew
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
  int i;                          /* Used to iterate through phrase terms */

  /* Components of the new expression object */
  Fts5Expr *pNew;
  Fts5ExprPhrase **apPhrase;
  Fts5ExprNode *pNode;
  Fts5ExprNearset *pNear;
  Fts5ExprPhrase *pCopy;


  pOrig = pExpr->apExprPhrase[iPhrase];
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );






  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
  );

  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
  }

  if( rc==SQLITE_OK ){
    /* All the allocations succeeded. Put the expression object together. */
    pNew->pIndex = pExpr->pIndex;
................................................................................
    pNear->nPhrase = 1;
    pNear->apPhrase[0] = pCopy;

    pCopy->nTerm = pOrig->nTerm;
    pCopy->pNode = pNode;
  }else{
    /* At least one allocation failed. Free them all. */
    if( pCopy ){
      for(i=0; i<pOrig->nTerm; i++){
        sqlite3_free(pCopy->aTerm[i].zTerm);
      }
      sqlite3_free(pCopy);
      sqlite3_free(pNear);
      sqlite3_free(pNode);
      sqlite3_free(apPhrase);
................................................................................
  int bMatch;

  assert( pNear->nPhrase>1 );

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5LookaheadReader) * pNear->nPhrase;
    a = (Fts5NearTrimmer*)sqlite3_malloc(nByte);
    if( !a ) return SQLITE_NOMEM;
    memset(a, 0, nByte);
  }else{
    memset(aStatic, 0, sizeof(aStatic));
  }

................................................................................
  int rc = SQLITE_OK;
  Fts5ExprNearset *pNear = pNode->pNear;
  while( 1 ){
    int i;

    /* Advance the iterators until they all point to the same rowid */
    rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
    if( pNode->bEof || rc!=SQLITE_OK ) break;

    /* Check that each phrase in the nearset matches the current row.
    ** Populate the pPhrase->poslist buffers at the same time. If any
    ** phrase is not a match, break out of the loop early.  */
    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
      if( pPhrase->nTerm>1 || pNear->iCol>=0 ){







<
|
|
>

<


>







 







|
<
<
<
<
<
<
<
|
>





>
>
>
>
>
>







|







 







<







 







|







 







|







100
101
102
103
104
105
106

107
108
109
110

111
112
113
114
115
116
117
118
119
120
...
265
266
267
268
269
270
271
272







273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
...
310
311
312
313
314
315
316

317
318
319
320
321
322
323
...
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
...
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
  int rc;
  int nPhrase;                    /* Size of apPhrase array */
  Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
  Fts5ExprNode *pExpr;            /* Result of a successful parse */
};

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){

  va_list ap;
  va_start(ap, zFmt);
  if( pParse->rc==SQLITE_OK ){
    pParse->zErr = sqlite3_vmprintf(zFmt, ap);

    pParse->rc = SQLITE_ERROR;
  }
  va_end(ap);
}

static int fts5ExprIsspace(char t){
  return t==' ' || t=='\t' || t=='\n' || t=='\r';
}

static int fts5ExprIstoken(char t){
................................................................................
int sqlite3Fts5ExprPhraseExpr(
  Fts5Config *pConfig,
  Fts5Expr *pExpr, 
  int iPhrase, 
  Fts5Expr **ppNew
){
  int rc = SQLITE_OK;             /* Return code */
  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */







  Fts5ExprPhrase *pCopy;          /* Copy of pOrig */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */

  pOrig = pExpr->apExprPhrase[iPhrase];
  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
  );
  if( pCopy ){
    int i;                          /* Used to iterate through phrase terms */
    Fts5ExprPhrase **apPhrase;
    Fts5ExprNode *pNode;
    Fts5ExprNearset *pNear;

    pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
    apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
    pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
    pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
        sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
    );

    for(i=0; i<pOrig->nTerm; i++){
      pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
      pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
    }

    if( rc==SQLITE_OK ){
      /* All the allocations succeeded. Put the expression object together. */
      pNew->pIndex = pExpr->pIndex;
................................................................................
      pNear->nPhrase = 1;
      pNear->apPhrase[0] = pCopy;

      pCopy->nTerm = pOrig->nTerm;
      pCopy->pNode = pNode;
    }else{
      /* At least one allocation failed. Free them all. */

      for(i=0; i<pOrig->nTerm; i++){
        sqlite3_free(pCopy->aTerm[i].zTerm);
      }
      sqlite3_free(pCopy);
      sqlite3_free(pNear);
      sqlite3_free(pNode);
      sqlite3_free(apPhrase);
................................................................................
  int bMatch;

  assert( pNear->nPhrase>1 );

  /* If the aStatic[] array is not large enough, allocate a large array
  ** using sqlite3_malloc(). This approach could be improved upon. */
  if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
    int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
    a = (Fts5NearTrimmer*)sqlite3_malloc(nByte);
    if( !a ) return SQLITE_NOMEM;
    memset(a, 0, nByte);
  }else{
    memset(aStatic, 0, sizeof(aStatic));
  }

................................................................................
  int rc = SQLITE_OK;
  Fts5ExprNearset *pNear = pNode->pNear;
  while( 1 ){
    int i;

    /* Advance the iterators until they all point to the same rowid */
    rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
    if( rc!=SQLITE_OK || pNode->bEof ) break;

    /* Check that each phrase in the nearset matches the current row.
    ** Populate the pPhrase->poslist buffers at the same time. If any
    ** phrase is not a match, break out of the loop early.  */
    for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
      Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
      if( pPhrase->nTerm>1 || pNear->iCol>=0 ){

Changes to ext/fts5/test/fts5ea.test.

5
6
7
8
9
10
11



12
13
14
15
16
17
18
..
36
37
38
39
40
41
42




43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65


66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#




source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ea

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
................................................................................
  4  {"abc def ghi" *}               {"abc" + "def" + "ghi" *}
  5  {one AND two}                   {"one" AND "two"}
  6  {one+two}                       {"one" + "two"}
  7  {one AND two OR three}          {("one" AND "two") OR "three"}
  8  {one OR two AND three}          {"one" OR ("two" AND "three")}
  9  {NEAR(one two)}                 {NEAR("one" "two", 10)}
  10 {NEAR("one three"* two, 5)}     {NEAR("one" + "three" * "two", 5)}




} {
  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}

foreach {tn expr res} {
  1 {c1:abc}                           
    {c1 : "abc"}
  2 {c2 : NEAR(one two) c1:"hello world"} 
    {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
} {
  do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
}

breakpoint
foreach {tn expr err} {
  1 {AND}                          {fts5: syntax error near "AND"}
  2 {abc def AND}                  {fts5: syntax error near ""}
  3 {abc OR AND}                   {fts5: syntax error near "AND"}
  4 {(a OR b) abc}                 {fts5: syntax error near "abc"}
  5 {NEaR (a b)}                   {fts5: syntax error near "NEaR"}
  6 {(a OR b) NOT c)}              {fts5: syntax error near ")"}
  7 {nosuch: a nosuch2: b}         {no such column: nosuch}
  8 {addr: a nosuch2: b}           {no such column: nosuch2}


} {
  do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
}



# do_syntax_error_test 1.0 {NOT} {syntax error near "NOT"}



# do_catchsql_test 1.1 { 
 #  SELECT fts5_expr('a OR b NOT c') 
#} {0 {"a" OR "b" NOT "c"}}


#do_execsql_test 1.0 { SELECT fts5_expr('a') } {{"a"}}

finish_test







>
>
>







 







>
>
>
>













<









>
>






<

<
<
<
<
<
<
<
<
<

5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
..
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

80









81
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
#
# Test the fts5 expression parser directly using the fts5_expr() SQL
# test function.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5ea

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
................................................................................
  4  {"abc def ghi" *}               {"abc" + "def" + "ghi" *}
  5  {one AND two}                   {"one" AND "two"}
  6  {one+two}                       {"one" + "two"}
  7  {one AND two OR three}          {("one" AND "two") OR "three"}
  8  {one OR two AND three}          {"one" OR ("two" AND "three")}
  9  {NEAR(one two)}                 {NEAR("one" "two", 10)}
  10 {NEAR("one three"* two, 5)}     {NEAR("one" + "three" * "two", 5)}
  11 {a OR b NOT c}                  {"a" OR ("b" NOT "c")}
  12 "\x20one\x20two\x20three"       {("one" AND "two") AND "three"}
  13 "\x09one\x0Atwo\x0Dthree"       {("one" AND "two") AND "three"}
  14 {"abc""def"}                    {"abc" + "def"}
} {
  do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}

foreach {tn expr res} {
  1 {c1:abc}                           
    {c1 : "abc"}
  2 {c2 : NEAR(one two) c1:"hello world"} 
    {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
} {
  do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
}


foreach {tn expr err} {
  1 {AND}                          {fts5: syntax error near "AND"}
  2 {abc def AND}                  {fts5: syntax error near ""}
  3 {abc OR AND}                   {fts5: syntax error near "AND"}
  4 {(a OR b) abc}                 {fts5: syntax error near "abc"}
  5 {NEaR (a b)}                   {fts5: syntax error near "NEaR"}
  6 {(a OR b) NOT c)}              {fts5: syntax error near ")"}
  7 {nosuch: a nosuch2: b}         {no such column: nosuch}
  8 {addr: a nosuch2: b}           {no such column: nosuch2}
  9 {NOT}                          {fts5: syntax error near "NOT"}
  10 {a AND "abc}                  {unterminated string}
} {
  do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
}














finish_test

Changes to ext/fts5/test/fts5fault4.test.

154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
...
199
200
201
202
203
204
205


206














207
208




209






































210
211
    rank MATCH 'rowidprefix(''$::str'')'
    LIMIT 1
  "
} -test {
  faultsim_test_result "0 {10-$::str {a b c}}"
}

}


#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x3 USING fts5(xxx);
................................................................................

do_faultsim_test 6.2 -faults oom-t* -body {
  db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' }
} -test {
  faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM}
}































































finish_test








<
<







 







>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|

>
>
>
>
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


154
155
156
157
158
159
160


161
162
163
164
165
166
167
...
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
    rank MATCH 'rowidprefix(''$::str'')'
    LIMIT 1
  "
} -test {
  faultsim_test_result "0 {10-$::str {a b c}}"
}




#-------------------------------------------------------------------------
# OOM errors within auxiliary functions.
#
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE x3 USING fts5(xxx);
................................................................................

do_faultsim_test 6.2 -faults oom-t* -body {
  db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' }
} -test {
  faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM}
}

#-------------------------------------------------------------------------
# OOM error when querying for a phrase with many tokens.
#
reset_db
do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE tt USING fts5(x, y);
  INSERT INTO tt VALUES('f b g b c b', 'f a d c c b');  -- 1
  INSERT INTO tt VALUES('d a e f e d', 'f b b d e e');  -- 2
  INSERT INTO tt VALUES('f b g a d c', 'e f c f a d');  -- 3
  INSERT INTO tt VALUES('f f c d g f', 'f a e b g b');  -- 4
  INSERT INTO tt VALUES('a g b d a g', 'e g a e a c');  -- 5
  INSERT INTO tt VALUES('c d b d e f', 'f g e g e e');  -- 6
  INSERT INTO tt VALUES('e g f f b c', 'f c e f g f');  -- 7
  INSERT INTO tt VALUES('e g c f c e', 'f e e a f g');  -- 8
  INSERT INTO tt VALUES('e a e b e e', 'd c c f f f');  -- 9
  INSERT INTO tt VALUES('f a g g c c', 'e g d g c e');  -- 10
  INSERT INTO tt VALUES('c d b a e f', 'f g e h e e');  -- 11
}

do_faultsim_test 7.2 -faults oom-* -body {
  db eval { SELECT rowid FROM tt WHERE tt MATCH 'f+g+e+g+e+e' }
} -test {
  faultsim_test_result {0 6} {1 SQLITE_NOMEM}
}

do_faultsim_test 7.3 -faults oom-* -body {
  db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d e f)' }
} -test {
  faultsim_test_result {0 11} {1 SQLITE_NOMEM}
}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE tt USING fts5(x);
  INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
  BEGIN;
    INSERT INTO tt(rowid, x) VALUES(1, 'a b c d x x');
    WITH ii(i) AS (SELECT 2 UNION ALL SELECT i+1 FROM ii WHERE i<99)
      INSERT INTO tt(rowid, x) SELECT i, 'a b c x x d' FROM ii;
    INSERT INTO tt(rowid, x) VALUES(100, 'a b c d x x');
  COMMIT;
}

do_faultsim_test 8.1 -faults oom-t* -body {
  db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d, 2)' }
} -test {
  faultsim_test_result {0 {1 100}} {1 SQLITE_NOMEM}
}

do_faultsim_test 8.2 -faults oom-t* -body {
  db eval { SELECT count(*) FROM tt WHERE tt MATCH 'a OR d' }
} -test {
  faultsim_test_result {0 100} {1 SQLITE_NOMEM}
}




finish_test