/ Check-in [d4331943]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further improvements to test coverage of fts5 code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: d4331943dff259380c4025bb740d8aba6972d351
User & Date: dan 2015-05-01 20:38:57
Context
2015-05-02
20:35
Reorganize some of the fts5 expression parsing code. Improve test coverage of the same. check-in: c4456dc5 user: dan tags: fts5
2015-05-01
20:38
Further improvements to test coverage of fts5 code. check-in: d4331943 user: dan tags: fts5
12:14
Improve test coverage of fts5.c. check-in: add4f468 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

   100    100     int rc;
   101    101     int nPhrase;                    /* Size of apPhrase array */
   102    102     Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
   103    103     Fts5ExprNode *pExpr;            /* Result of a successful parse */
   104    104   };
   105    105   
   106    106   void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
          107  +  va_list ap;
          108  +  va_start(ap, zFmt);
   107    109     if( pParse->rc==SQLITE_OK ){
   108         -    va_list ap;
   109         -    va_start(ap, zFmt);
   110    110       pParse->zErr = sqlite3_vmprintf(zFmt, ap);
   111         -    va_end(ap);
   112    111       pParse->rc = SQLITE_ERROR;
   113    112     }
          113  +  va_end(ap);
   114    114   }
   115    115   
   116    116   static int fts5ExprIsspace(char t){
   117    117     return t==' ' || t=='\t' || t=='\n' || t=='\r';
   118    118   }
   119    119   
   120    120   static int fts5ExprIstoken(char t){
................................................................................
   265    265   int sqlite3Fts5ExprPhraseExpr(
   266    266     Fts5Config *pConfig,
   267    267     Fts5Expr *pExpr, 
   268    268     int iPhrase, 
   269    269     Fts5Expr **ppNew
   270    270   ){
   271    271     int rc = SQLITE_OK;             /* Return code */
   272         -  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
   273         -  int i;                          /* Used to iterate through phrase terms */
   274         -
   275         -  /* Components of the new expression object */
   276         -  Fts5Expr *pNew;
   277         -  Fts5ExprPhrase **apPhrase;
   278         -  Fts5ExprNode *pNode;
   279         -  Fts5ExprNearset *pNear;
   280         -  Fts5ExprPhrase *pCopy;
          272  +  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
          273  +  Fts5ExprPhrase *pCopy;          /* Copy of pOrig */
          274  +  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
   281    275   
   282    276     pOrig = pExpr->apExprPhrase[iPhrase];
   283    277     pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
   284    278         sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
   285    279     );
   286         -  pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
   287         -  apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
   288         -  pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
   289         -  pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
   290         -      sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
   291         -  );
   292         -
   293         -  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
   294         -    pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
   295         -    pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
   296         -  }
   297         -
   298         -  if( rc==SQLITE_OK ){
   299         -    /* All the allocations succeeded. Put the expression object together. */
   300         -    pNew->pIndex = pExpr->pIndex;
   301         -    pNew->pRoot = pNode;
   302         -    pNew->nPhrase = 1;
   303         -    pNew->apExprPhrase = apPhrase;
   304         -    pNew->apExprPhrase[0] = pCopy;
   305         -
   306         -    pNode->eType = FTS5_STRING;
   307         -    pNode->pNear = pNear;
   308         -
   309         -    pNear->iCol = -1;
   310         -    pNear->nPhrase = 1;
   311         -    pNear->apPhrase[0] = pCopy;
   312         -
   313         -    pCopy->nTerm = pOrig->nTerm;
   314         -    pCopy->pNode = pNode;
   315         -  }else{
   316         -    /* At least one allocation failed. Free them all. */
   317         -    if( pCopy ){
          280  +  if( pCopy ){
          281  +    int i;                          /* Used to iterate through phrase terms */
          282  +    Fts5ExprPhrase **apPhrase;
          283  +    Fts5ExprNode *pNode;
          284  +    Fts5ExprNearset *pNear;
          285  +
          286  +    pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
          287  +    apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
          288  +    pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
          289  +    pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
          290  +        sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
          291  +    );
          292  +
          293  +    for(i=0; i<pOrig->nTerm; i++){
          294  +      pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);
          295  +      pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
          296  +    }
          297  +
          298  +    if( rc==SQLITE_OK ){
          299  +      /* All the allocations succeeded. Put the expression object together. */
          300  +      pNew->pIndex = pExpr->pIndex;
          301  +      pNew->pRoot = pNode;
          302  +      pNew->nPhrase = 1;
          303  +      pNew->apExprPhrase = apPhrase;
          304  +      pNew->apExprPhrase[0] = pCopy;
          305  +
          306  +      pNode->eType = FTS5_STRING;
          307  +      pNode->pNear = pNear;
          308  +
          309  +      pNear->iCol = -1;
          310  +      pNear->nPhrase = 1;
          311  +      pNear->apPhrase[0] = pCopy;
          312  +
          313  +      pCopy->nTerm = pOrig->nTerm;
          314  +      pCopy->pNode = pNode;
          315  +    }else{
          316  +      /* At least one allocation failed. Free them all. */
   318    317         for(i=0; i<pOrig->nTerm; i++){
   319    318           sqlite3_free(pCopy->aTerm[i].zTerm);
   320    319         }
   321    320         sqlite3_free(pCopy);
   322    321         sqlite3_free(pNear);
   323    322         sqlite3_free(pNode);
   324    323         sqlite3_free(apPhrase);
................................................................................
   500    499     int bMatch;
   501    500   
   502    501     assert( pNear->nPhrase>1 );
   503    502   
   504    503     /* If the aStatic[] array is not large enough, allocate a large array
   505    504     ** using sqlite3_malloc(). This approach could be improved upon. */
   506    505     if( pNear->nPhrase>(sizeof(aStatic) / sizeof(aStatic[0])) ){
   507         -    int nByte = sizeof(Fts5LookaheadReader) * pNear->nPhrase;
          506  +    int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
   508    507       a = (Fts5NearTrimmer*)sqlite3_malloc(nByte);
   509    508       if( !a ) return SQLITE_NOMEM;
   510    509       memset(a, 0, nByte);
   511    510     }else{
   512    511       memset(aStatic, 0, sizeof(aStatic));
   513    512     }
   514    513   
................................................................................
   715    714     int rc = SQLITE_OK;
   716    715     Fts5ExprNearset *pNear = pNode->pNear;
   717    716     while( 1 ){
   718    717       int i;
   719    718   
   720    719       /* Advance the iterators until they all point to the same rowid */
   721    720       rc = fts5ExprNearNextRowidMatch(pExpr, pNode, bFromValid, iFrom);
   722         -    if( pNode->bEof || rc!=SQLITE_OK ) break;
          721  +    if( rc!=SQLITE_OK || pNode->bEof ) break;
   723    722   
   724    723       /* Check that each phrase in the nearset matches the current row.
   725    724       ** Populate the pPhrase->poslist buffers at the same time. If any
   726    725       ** phrase is not a match, break out of the loop early.  */
   727    726       for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   728    727         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   729    728         if( pPhrase->nTerm>1 || pNear->iCol>=0 ){

Changes to ext/fts5/test/fts5ea.test.

     5      5   #
     6      6   #    May you do good and not evil.
     7      7   #    May you find forgiveness for yourself and forgive others.
     8      8   #    May you share freely, never taking more than you give.
     9      9   #
    10     10   #*************************************************************************
    11     11   #
           12  +# Test the fts5 expression parser directly using the fts5_expr() SQL
           13  +# test function.
           14  +#
    12     15   
    13     16   source [file join [file dirname [info script]] fts5_common.tcl]
    14     17   set testprefix fts5ea
    15     18   
    16     19   # If SQLITE_ENABLE_FTS5 is defined, omit this file.
    17     20   ifcapable !fts5 {
    18     21     finish_test
................................................................................
    36     39     4  {"abc def ghi" *}               {"abc" + "def" + "ghi" *}
    37     40     5  {one AND two}                   {"one" AND "two"}
    38     41     6  {one+two}                       {"one" + "two"}
    39     42     7  {one AND two OR three}          {("one" AND "two") OR "three"}
    40     43     8  {one OR two AND three}          {"one" OR ("two" AND "three")}
    41     44     9  {NEAR(one two)}                 {NEAR("one" "two", 10)}
    42     45     10 {NEAR("one three"* two, 5)}     {NEAR("one" + "three" * "two", 5)}
           46  +  11 {a OR b NOT c}                  {"a" OR ("b" NOT "c")}
           47  +  12 "\x20one\x20two\x20three"       {("one" AND "two") AND "three"}
           48  +  13 "\x09one\x0Atwo\x0Dthree"       {("one" AND "two") AND "three"}
           49  +  14 {"abc""def"}                    {"abc" + "def"}
    43     50   } {
    44     51     do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
    45     52   }
    46     53   
    47     54   foreach {tn expr res} {
    48     55     1 {c1:abc}                           
    49     56       {c1 : "abc"}
    50     57     2 {c2 : NEAR(one two) c1:"hello world"} 
    51     58       {c2 : NEAR("one" "two", 10) AND c1 : "hello" + "world"}
    52     59   } {
    53     60     do_execsql_test 2.$tn {SELECT fts5_expr($expr, 'c1', 'c2')} [list $res]
    54     61   }
    55     62   
    56         -breakpoint
    57     63   foreach {tn expr err} {
    58     64     1 {AND}                          {fts5: syntax error near "AND"}
    59     65     2 {abc def AND}                  {fts5: syntax error near ""}
    60     66     3 {abc OR AND}                   {fts5: syntax error near "AND"}
    61     67     4 {(a OR b) abc}                 {fts5: syntax error near "abc"}
    62     68     5 {NEaR (a b)}                   {fts5: syntax error near "NEaR"}
    63     69     6 {(a OR b) NOT c)}              {fts5: syntax error near ")"}
    64     70     7 {nosuch: a nosuch2: b}         {no such column: nosuch}
    65     71     8 {addr: a nosuch2: b}           {no such column: nosuch2}
           72  +  9 {NOT}                          {fts5: syntax error near "NOT"}
           73  +  10 {a AND "abc}                  {unterminated string}
    66     74   } {
    67     75     do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
    68     76   }
    69     77   
    70     78   
    71     79   
    72         -# do_syntax_error_test 1.0 {NOT} {syntax error near "NOT"}
    73         -
    74         -
    75         -
    76         -# do_catchsql_test 1.1 { 
    77         - #  SELECT fts5_expr('a OR b NOT c') 
    78         -#} {0 {"a" OR "b" NOT "c"}}
    79         -
    80         -
    81         -#do_execsql_test 1.0 { SELECT fts5_expr('a') } {{"a"}}
    82     80   
    83     81   finish_test

Changes to ext/fts5/test/fts5fault4.test.

   154    154       rank MATCH 'rowidprefix(''$::str'')'
   155    155       LIMIT 1
   156    156     "
   157    157   } -test {
   158    158     faultsim_test_result "0 {10-$::str {a b c}}"
   159    159   }
   160    160   
   161         -}
   162         -
   163    161   
   164    162   #-------------------------------------------------------------------------
   165    163   # OOM errors within auxiliary functions.
   166    164   #
   167    165   reset_db
   168    166   do_execsql_test 6.0 {
   169    167     CREATE VIRTUAL TABLE x3 USING fts5(xxx);
................................................................................
   199    197   
   200    198   do_faultsim_test 6.2 -faults oom-t* -body {
   201    199     db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' }
   202    200   } -test {
   203    201     faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM}
   204    202   }
   205    203   
          204  +#-------------------------------------------------------------------------
          205  +# OOM error when querying for a phrase with many tokens.
          206  +#
          207  +reset_db
          208  +do_execsql_test 7.0 {
          209  +  CREATE VIRTUAL TABLE tt USING fts5(x, y);
          210  +  INSERT INTO tt VALUES('f b g b c b', 'f a d c c b');  -- 1
          211  +  INSERT INTO tt VALUES('d a e f e d', 'f b b d e e');  -- 2
          212  +  INSERT INTO tt VALUES('f b g a d c', 'e f c f a d');  -- 3
          213  +  INSERT INTO tt VALUES('f f c d g f', 'f a e b g b');  -- 4
          214  +  INSERT INTO tt VALUES('a g b d a g', 'e g a e a c');  -- 5
          215  +  INSERT INTO tt VALUES('c d b d e f', 'f g e g e e');  -- 6
          216  +  INSERT INTO tt VALUES('e g f f b c', 'f c e f g f');  -- 7
          217  +  INSERT INTO tt VALUES('e g c f c e', 'f e e a f g');  -- 8
          218  +  INSERT INTO tt VALUES('e a e b e e', 'd c c f f f');  -- 9
          219  +  INSERT INTO tt VALUES('f a g g c c', 'e g d g c e');  -- 10
          220  +  INSERT INTO tt VALUES('c d b a e f', 'f g e h e e');  -- 11
          221  +}
          222  +
          223  +do_faultsim_test 7.2 -faults oom-* -body {
          224  +  db eval { SELECT rowid FROM tt WHERE tt MATCH 'f+g+e+g+e+e' }
          225  +} -test {
          226  +  faultsim_test_result {0 6} {1 SQLITE_NOMEM}
          227  +}
          228  +
          229  +do_faultsim_test 7.3 -faults oom-* -body {
          230  +  db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d e f)' }
          231  +} -test {
          232  +  faultsim_test_result {0 11} {1 SQLITE_NOMEM}
          233  +}
          234  +
          235  +}
          236  +
          237  +#-------------------------------------------------------------------------
          238  +#
          239  +reset_db
          240  +do_execsql_test 8.0 {
          241  +  CREATE VIRTUAL TABLE tt USING fts5(x);
          242  +  INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
          243  +  BEGIN;
          244  +    INSERT INTO tt(rowid, x) VALUES(1, 'a b c d x x');
          245  +    WITH ii(i) AS (SELECT 2 UNION ALL SELECT i+1 FROM ii WHERE i<99)
          246  +      INSERT INTO tt(rowid, x) SELECT i, 'a b c x x d' FROM ii;
          247  +    INSERT INTO tt(rowid, x) VALUES(100, 'a b c d x x');
          248  +  COMMIT;
          249  +}
          250  +
          251  +do_faultsim_test 8.1 -faults oom-t* -body {
          252  +  db eval { SELECT rowid FROM tt WHERE tt MATCH 'NEAR(a b c d, 2)' }
          253  +} -test {
          254  +  faultsim_test_result {0 {1 100}} {1 SQLITE_NOMEM}
          255  +}
          256  +
          257  +do_faultsim_test 8.2 -faults oom-t* -body {
          258  +  db eval { SELECT count(*) FROM tt WHERE tt MATCH 'a OR d' }
          259  +} -test {
          260  +  faultsim_test_result {0 100} {1 SQLITE_NOMEM}
          261  +}
   206    262   
   207    263   
   208    264   
   209    265   
   210    266   finish_test
   211    267