/ Check-in [c847543f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396
User & Date: dan 2017-04-12 17:50:12
Context
2017-04-13
00:12
Fix a regression caused by the fix for ticket [6c9b5514077fed34551f98e64c09a1] - control characters allowed in JSON. check-in: 8e7b6118 user: drh tags: trunk
2017-04-12
17:50
Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>". check-in: c847543f user: dan tags: trunk
17:38
Improved \n and \r escapes in the ext/misc/dbdump.c utility function. The implementation of dbdump.c now matches the implementation in the CLI. check-in: f2643315 user: drh tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   734    734   );
   735    735   
   736    736   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
   737    737   void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
   738    738   void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
   739    739   
   740    740   void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
   741         -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
          741  +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
   742    742   Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
   743    743   void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
   744    744   void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
   745    745   
   746    746   /*
   747    747   ** End of interface to code in fts5_expr.c.
   748    748   **************************************************************************/

Changes to ext/fts5/fts5_expr.c.

  1882   1882       assert( pParse->rc!=SQLITE_OK );
  1883   1883       sqlite3_free(pColset);
  1884   1884     }
  1885   1885   
  1886   1886     return pRet;
  1887   1887   }
  1888   1888   
         1889  +/*
         1890  +** If argument pOrig is NULL, or if (*pRc) is set to anything other than
         1891  +** SQLITE_OK when this function is called, NULL is returned. 
         1892  +**
         1893  +** Otherwise, a copy of (*pOrig) is made into memory obtained from
         1894  +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
         1895  +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
         1896  +*/
         1897  +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
         1898  +  Fts5Colset *pRet;
         1899  +  if( pOrig ){
         1900  +    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
         1901  +    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
         1902  +    if( pRet ){ 
         1903  +      memcpy(pRet, pOrig, nByte);
         1904  +    }
         1905  +  }else{
         1906  +    pRet = 0;
         1907  +  }
         1908  +  return pRet;
         1909  +}
         1910  +
         1911  +/*
         1912  +** Remove from colset pColset any columns that are not also in colset pMerge.
         1913  +*/
         1914  +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
         1915  +  int iIn = 0;          /* Next input in pColset */
         1916  +  int iMerge = 0;       /* Next input in pMerge */
         1917  +  int iOut = 0;         /* Next output slot in pColset */
         1918  +
         1919  +  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
         1920  +    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
         1921  +    if( iDiff==0 ){
         1922  +      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
         1923  +      iMerge++;
         1924  +      iIn++;
         1925  +    }else if( iDiff>0 ){
         1926  +      iMerge++;
         1927  +    }else{
         1928  +      iIn++;
         1929  +    }
         1930  +  }
         1931  +  pColset->nCol = iOut;
         1932  +}
         1933  +
         1934  +/*
         1935  +** Recursively apply colset pColset to expression node pNode and all of
         1936  +** its decendents. If (*ppFree) is not NULL, it contains a spare copy
         1937  +** of pColset. This function may use the spare copy and set (*ppFree) to
         1938  +** zero, or it may create copies of pColset using fts5CloneColset().
         1939  +*/
         1940  +static void fts5ParseSetColset(
         1941  +  Fts5Parse *pParse, 
         1942  +  Fts5ExprNode *pNode, 
         1943  +  Fts5Colset *pColset,
         1944  +  Fts5Colset **ppFree
         1945  +){
         1946  +  if( pParse->rc==SQLITE_OK ){
         1947  +    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
         1948  +         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
         1949  +         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
         1950  +    );
         1951  +    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
         1952  +      Fts5ExprNearset *pNear = pNode->pNear;
         1953  +      if( pNear->pColset ){
         1954  +        fts5MergeColset(pNear->pColset, pColset);
         1955  +        if( pNear->pColset->nCol==0 ){
         1956  +          pNode->eType = FTS5_EOF;
         1957  +          pNode->xNext = 0;
         1958  +        }
         1959  +      }else if( *ppFree ){
         1960  +        pNear->pColset = pColset;
         1961  +        *ppFree = 0;
         1962  +      }else{
         1963  +        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
         1964  +      }
         1965  +    }else{
         1966  +      int i;
         1967  +      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
         1968  +      for(i=0; i<pNode->nChild; i++){
         1969  +        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
         1970  +      }
         1971  +    }
         1972  +  }
         1973  +}
         1974  +
         1975  +/*
         1976  +** Apply colset pColset to expression node pExpr and all of its descendents.
         1977  +*/
  1889   1978   void sqlite3Fts5ParseSetColset(
  1890   1979     Fts5Parse *pParse, 
  1891         -  Fts5ExprNearset *pNear, 
         1980  +  Fts5ExprNode *pExpr, 
  1892   1981     Fts5Colset *pColset 
  1893   1982   ){
         1983  +  Fts5Colset *pFree = pColset;
  1894   1984     if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
  1895   1985       pParse->rc = SQLITE_ERROR;
  1896   1986       pParse->zErr = sqlite3_mprintf(
  1897   1987         "fts5: column queries are not supported (detail=none)"
  1898   1988       );
  1899         -    sqlite3_free(pColset);
  1900         -    return;
  1901         -  }
  1902         -
  1903         -  if( pNear ){
  1904         -    pNear->pColset = pColset;
  1905   1989     }else{
  1906         -    sqlite3_free(pColset);
         1990  +    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  1907   1991     }
         1992  +  sqlite3_free(pFree);
  1908   1993   }
  1909   1994   
  1910   1995   static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  1911   1996     switch( pNode->eType ){
  1912   1997       case FTS5_STRING: {
  1913   1998         Fts5ExprNearset *pNear = pNode->pNear;
  1914   1999         if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 

Changes to ext/fts5/fts5_index.c.

  3154   3154     while( p<pEnd && *p!=0x01 ){
  3155   3155       while( *p++ & 0x80 );
  3156   3156     }
  3157   3157   
  3158   3158     return p - (*pa);
  3159   3159   }
  3160   3160   
  3161         -static int fts5IndexExtractColset (
         3161  +static void fts5IndexExtractColset(
         3162  +  int *pRc,
  3162   3163     Fts5Colset *pColset,            /* Colset to filter on */
  3163   3164     const u8 *pPos, int nPos,       /* Position list */
  3164   3165     Fts5Buffer *pBuf                /* Output buffer */
  3165   3166   ){
  3166         -  int rc = SQLITE_OK;
  3167         -  int i;
  3168         -
  3169         -  fts5BufferZero(pBuf);
  3170         -  for(i=0; i<pColset->nCol; i++){
  3171         -    const u8 *pSub = pPos;
  3172         -    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
  3173         -    if( nSub ){
  3174         -      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         3167  +  if( *pRc==SQLITE_OK ){
         3168  +    int i;
         3169  +    fts5BufferZero(pBuf);
         3170  +    for(i=0; i<pColset->nCol; i++){
         3171  +      const u8 *pSub = pPos;
         3172  +      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         3173  +      if( nSub ){
         3174  +        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
         3175  +      }
  3175   3176       }
  3176   3177     }
  3177         -  return rc;
  3178   3178   }
  3179   3179   
  3180   3180   /*
  3181   3181   ** xSetOutputs callback used by detail=none tables.
  3182   3182   */
  3183   3183   static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
  3184   3184     assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
  3294   3294       /* All data is stored on the current page. Populate the output 
  3295   3295       ** variables to point into the body of the page object. */
  3296   3296       const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  3297   3297       if( pColset->nCol==1 ){
  3298   3298         pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
  3299   3299         pIter->base.pData = a;
  3300   3300       }else{
         3301  +      int *pRc = &pIter->pIndex->rc;
  3301   3302         fts5BufferZero(&pIter->poslist);
  3302         -      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         3303  +      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
  3303   3304         pIter->base.pData = pIter->poslist.p;
  3304   3305         pIter->base.nData = pIter->poslist.n;
  3305   3306       }
  3306   3307     }else{
  3307   3308       /* The data is distributed over two or more pages. Copy it into the
  3308   3309       ** Fts5Iter.poslist buffer and then set the output pointer to point
  3309   3310       ** to this buffer.  */

Changes to ext/fts5/fts5parse.y.

    85     85   %type cnearset    {Fts5ExprNode*}
    86     86   %type expr        {Fts5ExprNode*}
    87     87   %type exprlist    {Fts5ExprNode*}
    88     88   %destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
    89     89   %destructor expr     { sqlite3Fts5ParseNodeFree($$); }
    90     90   %destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
    91     91   
    92         -expr(A) ::= expr(X) AND expr(Y). {
    93         -  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
    94         -}
    95         -expr(A) ::= expr(X) OR expr(Y). {
    96         -  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
    97         -}
    98         -expr(A) ::= expr(X) NOT expr(Y). {
    99         -  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
   100         -}
   101         -
   102         -expr(A) ::= LP expr(X) RP. {A = X;}
   103         -expr(A) ::= exprlist(X).   {A = X;}
   104         -
   105         -exprlist(A) ::= cnearset(X). {A = X;}
   106         -exprlist(A) ::= exprlist(X) cnearset(Y). {
   107         -  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
   108         -}
   109         -
   110         -cnearset(A) ::= nearset(X). { 
   111         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
   112         -}
   113         -cnearset(A) ::= colset(X) COLON nearset(Y). { 
   114         -  sqlite3Fts5ParseSetColset(pParse, Y, X);
   115         -  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
   116         -}
   117         -
   118     92   %type colset {Fts5Colset*}
   119     93   %destructor colset { sqlite3_free($$); }
   120     94   %type colsetlist {Fts5Colset*}
   121     95   %destructor colsetlist { sqlite3_free($$); }
   122     96   
   123     97   colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
   124     98       A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
   133    107   }
   134    108   
   135    109   colsetlist(A) ::= colsetlist(Y) STRING(X). { 
   136    110     A = sqlite3Fts5ParseColset(pParse, Y, &X); }
   137    111   colsetlist(A) ::= STRING(X). { 
   138    112     A = sqlite3Fts5ParseColset(pParse, 0, &X); 
   139    113   }
          114  +
          115  +expr(A) ::= expr(X) AND expr(Y). {
          116  +  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
          117  +}
          118  +expr(A) ::= expr(X) OR expr(Y). {
          119  +  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
          120  +}
          121  +expr(A) ::= expr(X) NOT expr(Y). {
          122  +  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
          123  +}
          124  +
          125  +expr(A) ::= colset(X) COLON LP expr(Y) RP. {
          126  +  sqlite3Fts5ParseSetColset(pParse, Y, X);
          127  +  A = Y;
          128  +}
          129  +expr(A) ::= LP expr(X) RP. {A = X;}
          130  +expr(A) ::= exprlist(X).   {A = X;}
          131  +
          132  +exprlist(A) ::= cnearset(X). {A = X;}
          133  +exprlist(A) ::= exprlist(X) cnearset(Y). {
          134  +  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
          135  +}
          136  +
          137  +cnearset(A) ::= nearset(X). { 
          138  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
          139  +}
          140  +cnearset(A) ::= colset(X) COLON nearset(Y). { 
          141  +  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
          142  +  sqlite3Fts5ParseSetColset(pParse, A, X);
          143  +}
          144  +
   140    145   
   141    146   %type nearset     {Fts5ExprNearset*}
   142    147   %type nearphrases {Fts5ExprNearset*}
   143    148   %destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
   144    149   %destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
   145    150   
   146    151   nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }

Changes to ext/fts5/test/fts5colset.test.

    40     40       5 " - {d d c} : a" {1 2}
    41     41       6 "- {d c b a} : a" {}
    42     42       7 "-{\"a\"} : b" {1 2 3}
    43     43       8 "- c : a" {1 2 4}
    44     44       9 "-c : a"  {1 2 4}
    45     45       10 "-\"c\" : a"  {1 2 4}
    46     46     } {
    47         -  breakpoint
    48     47       do_execsql_test 1.$tn {
    49     48         SELECT rowid FROM t1($q)
    50     49       } $res
    51     50     }
    52     51   
    53         -
           52  +  foreach {tn q res} {
           53  +    0 {{a} : (a AND ":")}     {}
           54  +    1 "{a b c} : (a AND d)"   {2 3}
           55  +    2 "{a b c} : (a AND b:d)" {3}
           56  +    3 "{a b c} : (a AND d:d)" {}
           57  +    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
           58  +    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
           59  +    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
           60  +    7 "{a b c} : (b:a AND c:b)" {2}
           61  +  } {
           62  +    do_execsql_test 2.$tn {
           63  +      SELECT rowid FROM t1($q)
           64  +    } $res
           65  +  }
    54     66   }
    55     67   
    56     68   
    57     69   finish_test
    58     70   
    59     71   

Changes to ext/fts5/test/fts5faultB.test.

   102    102   do_faultsim_test 3.3 -faults oom* -body {
   103    103     execsql {
   104    104       SELECT rowid FROM x1('c') WHERE rowid>1;
   105    105     }
   106    106   } -test {
   107    107     faultsim_test_result {0 {2 3}}
   108    108   }
          109  +
          110  +#-------------------------------------------------------------------------
          111  +# Test OOM injection with nested colsets.
          112  +#
          113  +reset_db
          114  +do_execsql_test 4.0 {
          115  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
          116  +  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
          117  +  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
          118  +  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
          119  +  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
          120  +}
          121  +do_faultsim_test 4.1 -faults oom* -body {
          122  +  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
          123  +} -test {
          124  +  faultsim_test_result {0 2}
          125  +}
          126  +
          127  +do_faultsim_test 4.2 -faults oom* -body {
          128  +  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
          129  +} -test {
          130  +  faultsim_test_result {0 {2 3}}
          131  +}
          132  +
   109    133   
   110    134   finish_test
   111    135