/ Check-in [c847543f]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256:c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396
User & Date: dan 2017-04-12 17:50:12
Context
2017-04-13
00:12
Fix a regression caused by the fix for ticket [6c9b5514077fed34551f98e64c09a1] - control characters allowed in JSON. check-in: 8e7b6118 user: drh tags: trunk
2017-04-12
17:50
Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well as "<colset> : NEAR(...)" and "<colset> : <phrase>". check-in: c847543f user: dan tags: trunk
17:38
Improved \n and \r escapes in the ext/misc/dbdump.c utility function. The implementation of dbdump.c now matches the implementation in the CLI. check-in: f2643315 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
);

void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/







|







734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
);

void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/

Changes to ext/fts5/fts5_expr.c.

1882
1883
1884
1885
1886
1887
1888

























































































1889
1890
1891
1892
1893

1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906

1907

1908
1909
1910
1911
1912
1913
1914
    assert( pParse->rc!=SQLITE_OK );
    sqlite3_free(pColset);
  }

  return pRet;
}


























































































void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
){

  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );
    sqlite3_free(pColset);
    return;
  }

  if( pNear ){
    pNear->pColset = pColset;
  }else{
    sqlite3_free(pColset);

  }

}

static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  switch( pNode->eType ){
    case FTS5_STRING: {
      Fts5ExprNearset *pNear = pNode->pNear;
      if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|


>





<
<
<
<
<
<

<
>

>







1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988






1989

1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
    assert( pParse->rc!=SQLITE_OK );
    sqlite3_free(pColset);
  }

  return pRet;
}

/*
** If argument pOrig is NULL, or if (*pRc) is set to anything other than
** SQLITE_OK when this function is called, NULL is returned. 
**
** Otherwise, a copy of (*pOrig) is made into memory obtained from
** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
*/
static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
  Fts5Colset *pRet;
  if( pOrig ){
    int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
    if( pRet ){ 
      memcpy(pRet, pOrig, nByte);
    }
  }else{
    pRet = 0;
  }
  return pRet;
}

/*
** Remove from colset pColset any columns that are not also in colset pMerge.
*/
static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
  int iIn = 0;          /* Next input in pColset */
  int iMerge = 0;       /* Next input in pMerge */
  int iOut = 0;         /* Next output slot in pColset */

  while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
    int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
    if( iDiff==0 ){
      pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
      iMerge++;
      iIn++;
    }else if( iDiff>0 ){
      iMerge++;
    }else{
      iIn++;
    }
  }
  pColset->nCol = iOut;
}

/*
** Recursively apply colset pColset to expression node pNode and all of
** its decendents. If (*ppFree) is not NULL, it contains a spare copy
** of pColset. This function may use the spare copy and set (*ppFree) to
** zero, or it may create copies of pColset using fts5CloneColset().
*/
static void fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNode *pNode, 
  Fts5Colset *pColset,
  Fts5Colset **ppFree
){
  if( pParse->rc==SQLITE_OK ){
    assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING 
         || pNode->eType==FTS5_AND  || pNode->eType==FTS5_OR
         || pNode->eType==FTS5_NOT  || pNode->eType==FTS5_EOF
    );
    if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
      Fts5ExprNearset *pNear = pNode->pNear;
      if( pNear->pColset ){
        fts5MergeColset(pNear->pColset, pColset);
        if( pNear->pColset->nCol==0 ){
          pNode->eType = FTS5_EOF;
          pNode->xNext = 0;
        }
      }else if( *ppFree ){
        pNear->pColset = pColset;
        *ppFree = 0;
      }else{
        pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
      }
    }else{
      int i;
      assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
      for(i=0; i<pNode->nChild; i++){
        fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
      }
    }
  }
}

/*
** Apply colset pColset to expression node pExpr and all of its descendents.
*/
void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNode *pExpr, 
  Fts5Colset *pColset 
){
  Fts5Colset *pFree = pColset;
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
    pParse->zErr = sqlite3_mprintf(
      "fts5: column queries are not supported (detail=none)"
    );






  }else{

    fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
  }
  sqlite3_free(pFree);
}

static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  switch( pNode->eType ){
    case FTS5_STRING: {
      Fts5ExprNearset *pNear = pNode->pNear;
      if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 

Changes to ext/fts5/fts5_index.c.

3154
3155
3156
3157
3158
3159
3160
3161

3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177

3178
3179
3180
3181
3182
3183
3184
....
3294
3295
3296
3297
3298
3299
3300

3301
3302
3303
3304
3305
3306
3307
3308
3309
  while( p<pEnd && *p!=0x01 ){
    while( *p++ & 0x80 );
  }

  return p - (*pa);
}

static int fts5IndexExtractColset (

  Fts5Colset *pColset,            /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
){
  int rc = SQLITE_OK;
  int i;

  fts5BufferZero(pBuf);
  for(i=0; i<pColset->nCol; i++){
    const u8 *pSub = pPos;
    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
    if( nSub ){
      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
    }
  }
  return rc;

}

/*
** xSetOutputs callback used by detail=none tables.
*/
static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
    /* All data is stored on the current page. Populate the output 
    ** variables to point into the body of the page object. */
    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
    if( pColset->nCol==1 ){
      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
      pIter->base.pData = a;
    }else{

      fts5BufferZero(&pIter->poslist);
      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
      pIter->base.pData = pIter->poslist.p;
      pIter->base.nData = pIter->poslist.n;
    }
  }else{
    /* The data is distributed over two or more pages. Copy it into the
    ** Fts5Iter.poslist buffer and then set the output pointer to point
    ** to this buffer.  */







|
>




|
|
<
|
|
|
|
|
|
|
|
<
>







 







>

|







3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168

3169
3170
3171
3172
3173
3174
3175
3176

3177
3178
3179
3180
3181
3182
3183
3184
....
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
  while( p<pEnd && *p!=0x01 ){
    while( *p++ & 0x80 );
  }

  return p - (*pa);
}

static void fts5IndexExtractColset(
  int *pRc,
  Fts5Colset *pColset,            /* Colset to filter on */
  const u8 *pPos, int nPos,       /* Position list */
  Fts5Buffer *pBuf                /* Output buffer */
){
  if( *pRc==SQLITE_OK ){
    int i;

    fts5BufferZero(pBuf);
    for(i=0; i<pColset->nCol; i++){
      const u8 *pSub = pPos;
      int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
      if( nSub ){
        fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
      }
    }

  }
}

/*
** xSetOutputs callback used by detail=none tables.
*/
static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
................................................................................
    /* All data is stored on the current page. Populate the output 
    ** variables to point into the body of the page object. */
    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
    if( pColset->nCol==1 ){
      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
      pIter->base.pData = a;
    }else{
      int *pRc = &pIter->pIndex->rc;
      fts5BufferZero(&pIter->poslist);
      fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
      pIter->base.pData = pIter->poslist.p;
      pIter->base.nData = pIter->poslist.n;
    }
  }else{
    /* The data is distributed over two or more pages. Copy it into the
    ** Fts5Iter.poslist buffer and then set the output pointer to point
    ** to this buffer.  */

Changes to ext/fts5/fts5parse.y.

85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
...
133
134
135
136
137
138
139































140
141
142
143
144
145
146
%type cnearset    {Fts5ExprNode*}
%type expr        {Fts5ExprNode*}
%type exprlist    {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr     { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }

expr(A) ::= expr(X) AND expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}

expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X).   {A = X;}

exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}

cnearset(A) ::= nearset(X). { 
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
}
cnearset(A) ::= colset(X) COLON nearset(Y). { 
  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
}

%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }

colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
    A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
}

colsetlist(A) ::= colsetlist(Y) STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
}
































%type nearset     {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }

nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }







<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







 







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







85
86
87
88
89
90
91


























92
93
94
95
96
97
98
...
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
%type cnearset    {Fts5ExprNode*}
%type expr        {Fts5ExprNode*}
%type exprlist    {Fts5ExprNode*}
%destructor cnearset { sqlite3Fts5ParseNodeFree($$); }
%destructor expr     { sqlite3Fts5ParseNodeFree($$); }
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }



























%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }

colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
    A = sqlite3Fts5ParseColsetInvert(pParse, X);
................................................................................
}

colsetlist(A) ::= colsetlist(Y) STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
}

expr(A) ::= expr(X) AND expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
}
expr(A) ::= expr(X) OR expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
}
expr(A) ::= expr(X) NOT expr(Y). {
  A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
}

expr(A) ::= colset(X) COLON LP expr(Y) RP. {
  sqlite3Fts5ParseSetColset(pParse, Y, X);
  A = Y;
}
expr(A) ::= LP expr(X) RP. {A = X;}
expr(A) ::= exprlist(X).   {A = X;}

exprlist(A) ::= cnearset(X). {A = X;}
exprlist(A) ::= exprlist(X) cnearset(Y). {
  A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
}

cnearset(A) ::= nearset(X). { 
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); 
}
cnearset(A) ::= colset(X) COLON nearset(Y). { 
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
  sqlite3Fts5ParseSetColset(pParse, A, X);
}


%type nearset     {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }

nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }

Changes to ext/fts5/test/fts5colset.test.

40
41
42
43
44
45
46
47
48
49
50
51
52













53
54
55
56
57
58
59
    5 " - {d d c} : a" {1 2}
    6 "- {d c b a} : a" {}
    7 "-{\"a\"} : b" {1 2 3}
    8 "- c : a" {1 2 4}
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {
  breakpoint
    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)
    } $res
  }















}


finish_test









<





>
>
>
>
>
>
>
>
>
>
>
>
>
|






40
41
42
43
44
45
46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
    5 " - {d d c} : a" {1 2}
    6 "- {d c b a} : a" {}
    7 "-{\"a\"} : b" {1 2 3}
    8 "- c : a" {1 2 4}
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {

    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)
    } $res
  }

  foreach {tn q res} {
    0 {{a} : (a AND ":")}     {}
    1 "{a b c} : (a AND d)"   {2 3}
    2 "{a b c} : (a AND b:d)" {3}
    3 "{a b c} : (a AND d:d)" {}
    4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
    5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
    6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
    7 "{a b c} : (b:a AND c:b)" {2}
  } {
    do_execsql_test 2.$tn {
      SELECT rowid FROM t1($q)
    } $res
  }
}


finish_test


Changes to ext/fts5/test/fts5faultB.test.

102
103
104
105
106
107
108
109
























110
111
do_faultsim_test 3.3 -faults oom* -body {
  execsql {
    SELECT rowid FROM x1('c') WHERE rowid>1;
  }
} -test {
  faultsim_test_result {0 {2 3}}
}

























finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
do_faultsim_test 3.3 -faults oom* -body {
  execsql {
    SELECT rowid FROM x1('c') WHERE rowid>1;
  }
} -test {
  faultsim_test_result {0 {2 3}}
}

#-------------------------------------------------------------------------
# Test OOM injection with nested colsets.
#
reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
  INSERT INTO t1 VALUES('a', 'b', 'c', 'd');  -- 1
  INSERT INTO t1 VALUES('d', 'a', 'b', 'c');  -- 2
  INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
  INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
}
do_faultsim_test 4.1 -faults oom* -body {
  execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
} -test {
  faultsim_test_result {0 2}
}

do_faultsim_test 4.2 -faults oom* -body {
  execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
} -test {
  faultsim_test_result {0 {2 3}}
}


finish_test