SQLite

Check-in [14864f2b84]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Minor update to the way fts5 column filters are parsed.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 14864f2b8470fe98dbd17f59963bf1be8d4962f9
User & Date: dan 2016-08-09 19:48:37.555
Context
2016-08-09
21:01
Fix harmless compiler warning. (check-in: 9a5a4f6e3b user: drh tags: trunk)
19:48
Minor update to the way fts5 column filters are parsed. (check-in: 14864f2b84 user: dan tags: trunk)
19:26
Have fts5 interpret column lists that begin with a "-" character as "match any column except" lists. (check-in: e517545650 user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5Int.h.
734
735
736
737
738
739
740

741
742
743
744
745
746
747
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
void sqlite3Fts5ParseColsetNegative(Fts5Parse*, int);

void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/








>







734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*);
void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);

void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
void sqlite3Fts5ParseColsetNegative(Fts5Parse*, int);
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);

/*
** End of interface to code in fts5_expr.c.
**************************************************************************/

Changes to ext/fts5/fts5_expr.c.
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
** Parse context.
*/
struct Fts5Parse {
  Fts5Config *pConfig;
  char *zErr;
  int rc;
  int nPhrase;                    /* Size of apPhrase array */
  int bNegativeCollist;           /* Column list being parsed started with - */
  Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
  Fts5ExprNode *pExpr;            /* Result of a successful parse */
};

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
  va_list ap;
  va_start(ap, zFmt);







<







120
121
122
123
124
125
126

127
128
129
130
131
132
133
** Parse context.
*/
struct Fts5Parse {
  Fts5Config *pConfig;
  char *zErr;
  int rc;
  int nPhrase;                    /* Size of apPhrase array */

  Fts5ExprPhrase **apPhrase;      /* Array of all phrases */
  Fts5ExprNode *pExpr;            /* Result of a successful parse */
};

void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
  va_list ap;
  va_start(ap, zFmt);
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805

1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820

1821
1822
1823
1824

1825
1826

1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841

1842
1843
1844
1845
1846
1847
1848
#endif
  }

  return pNew;
}

/*
** The second argument passed to this function may be NULL, or it may be
** an existing Fts5Colset object. If it is passed NULL, this function
** returns a pointer to a new Fts5Colset object containing entries for
** all table columns except column iCol. If an OOM error occurs trying to
** allocate the Fts5Colset object, an error code is stored in pParse and 
** NULL returned.
**

** If the second argument is not NULL, a copy of it is returned. Before
** returning, any entry for column iCol is removed. It is not an error
** if the Fts5Colset object does not contain an entry for column iCol
** when this function is called.
*/
static Fts5Colset *fts5ParseNegativeColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *p,                  /* Existing colset object */
  int iCol                        /* New column to add to colset object */
){
  int i;
  Fts5Colset *pRet = p;

  if( pRet==0 ){
    int nCol = pParse->pConfig->nCol;

    pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc,
        sizeof(Fts5Colset) + sizeof(int)*nCol
    );
    if( pRet==0 ) return 0;

    pRet->nCol = nCol;
    for(i=0; i<nCol; i++){

      pRet->aiCol[i] = i;
    }
  }

  for(i=0; i<pRet->nCol; i++){
    if( pRet->aiCol[i]==iCol ){
      int nByte = sizeof(int)*(pRet->nCol-i-1);
      if( nByte ){
        memmove(&pRet->aiCol[i], &pRet->aiCol[i+1], nByte);
      }
      pRet->nCol--;
      break;
    }
  }


  return pRet;
}

Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *pColset,            /* Existing colset object */
  Fts5Token *p







<
<
|
<
<
<
<
>
|
<
<
<

<
|
<
<
<
<
|
<
<
|
>
|
|
|
|
>
|

>
|
|
<
|
<
<
<
<
<

<
<



>







1791
1792
1793
1794
1795
1796
1797


1798




1799
1800



1801

1802




1803


1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815

1816





1817


1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
#endif
  }

  return pNew;
}

/*


** Allocate and return an Fts5Colset object specifying the inverse of




** the colset passed as the second argument. Free the colset passed
** as the second argument before returning.



*/

Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){




  Fts5Colset *pRet;


  int nCol = pParse->pConfig->nCol;

  pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, 
      sizeof(Fts5Colset) + sizeof(int)*nCol
  );
  if( pRet ){
    int i;
    int iOld = 0;
    for(i=0; i<nCol; i++){
      if( iOld>=p->nCol || p->aiCol[iOld]!=i ){
        pRet->aiCol[pRet->nCol++] = i;
      }else{

        iOld++;





      }


    }
  }

  sqlite3_free(p);
  return pRet;
}

Fts5Colset *sqlite3Fts5ParseColset(
  Fts5Parse *pParse,              /* Store SQLITE_NOMEM here if required */
  Fts5Colset *pColset,            /* Existing colset object */
  Fts5Token *p
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
    Fts5Config *pConfig = pParse->pConfig;
    sqlite3Fts5Dequote(z);
    for(iCol=0; iCol<pConfig->nCol; iCol++){
      if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break;
    }
    if( iCol==pConfig->nCol ){
      sqlite3Fts5ParseError(pParse, "no such column: %s", z);
    }else if( pParse->bNegativeCollist ){
      pRet = fts5ParseNegativeColset(pParse, pColset, iCol);
    }else{
      pRet = fts5ParseColset(pParse, pColset, iCol);
    }
    sqlite3_free(z);
  }

  if( pRet==0 ){
    assert( pParse->rc!=SQLITE_OK );
    sqlite3_free(pColset);
  }

  return pRet;
}

/*
** Set (bVal==1) or clear (bVal==0) the Fts5Parse.bNegativeCollist flag.
**
** The parser calls this function as it begins to parse a colset (Fts5Colset
** object) with bVal set to 1 if the colset begins with a "-" or 0 otherwise.
*/
void sqlite3Fts5ParseColsetNegative(Fts5Parse *pParse, int bVal){
  pParse->bNegativeCollist = bVal;
}

void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
){
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;







<
<














<
<
<
<
<
<
<
<
<
<







1836
1837
1838
1839
1840
1841
1842


1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856










1857
1858
1859
1860
1861
1862
1863
    Fts5Config *pConfig = pParse->pConfig;
    sqlite3Fts5Dequote(z);
    for(iCol=0; iCol<pConfig->nCol; iCol++){
      if( 0==sqlite3_stricmp(pConfig->azCol[iCol], z) ) break;
    }
    if( iCol==pConfig->nCol ){
      sqlite3Fts5ParseError(pParse, "no such column: %s", z);


    }else{
      pRet = fts5ParseColset(pParse, pColset, iCol);
    }
    sqlite3_free(z);
  }

  if( pRet==0 ){
    assert( pParse->rc!=SQLITE_OK );
    sqlite3_free(pColset);
  }

  return pRet;
}











void sqlite3Fts5ParseSetColset(
  Fts5Parse *pParse, 
  Fts5ExprNearset *pNear, 
  Fts5Colset *pColset 
){
  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
    pParse->rc = SQLITE_ERROR;
Changes to ext/fts5/fts5parse.y.
115
116
117
118
119
120
121
122


123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
}

%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }
%type minus_opt {int}



colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= MINUS STRING(X). {
  sqlite3Fts5ParseColsetNegative(pParse, 1);
  A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colset(A) ::= STRING(X). {
  sqlite3Fts5ParseColsetNegative(pParse, 0);
  A = sqlite3Fts5ParseColset(pParse, 0, &X);
}

colsetlist(A) ::= colsetlist(Y) STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= minus_opt(M) STRING(X). { 
  sqlite3Fts5ParseColsetNegative(pParse, M);
  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
}

minus_opt(A) ::= MINUS. { A = 1; }
minus_opt(A) ::= .      { A = 0; }

%type nearset     {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }

nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {







|
>
>
|

|
<


|
|
|




|
<



<
<
<







115
116
117
118
119
120
121
122
123
124
125
126
127

128
129
130
131
132
133
134
135
136
137

138
139
140



141
142
143
144
145
146
147
  A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); 
}

%type colset {Fts5Colset*}
%destructor colset { sqlite3_free($$); }
%type colsetlist {Fts5Colset*}
%destructor colsetlist { sqlite3_free($$); }

colset(A) ::= MINUS LCP colsetlist(X) RCP. { 
    A = sqlite3Fts5ParseColsetInvert(pParse, X);
}
colset(A) ::= LCP colsetlist(X) RCP. { A = X; }
colset(A) ::= STRING(X). {

  A = sqlite3Fts5ParseColset(pParse, 0, &X);
}
colset(A) ::= MINUS STRING(X). {
  A = sqlite3Fts5ParseColset(pParse, 0, &X);
  A = sqlite3Fts5ParseColsetInvert(pParse, A);
}

colsetlist(A) ::= colsetlist(Y) STRING(X). { 
  A = sqlite3Fts5ParseColset(pParse, Y, &X); }
colsetlist(A) ::= STRING(X). { 

  A = sqlite3Fts5ParseColset(pParse, 0, &X); 
}




%type nearset     {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }

nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
Changes to ext/fts5/test/fts5colset.test.
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
    INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
    INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
  }

  foreach {tn q res} {
    1 "a"          {1 2 3 4}
    2 "{a}   : a"  {1}
    3 "{-a}   : a" {2 3 4}
    4 "{-a c} : a" {2 4}
    5 "{-d d c} : a" {1 2}
    6 "{-d c b a} : a" {}
    7 "{-\"a\"} : b" {1 2 3}
    8 "- c : a" {1 2 4}
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {
  breakpoint
    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)







|
|
|
|
|







31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
    INSERT INTO t1 VALUES('c', 'd', 'a', 'b');  -- 3
    INSERT INTO t1 VALUES('b', 'c', 'd', 'a');  -- 4
  }

  foreach {tn q res} {
    1 "a"          {1 2 3 4}
    2 "{a}   : a"  {1}
    3 "-{a}   : a" {2 3 4}
    4 "- {a c} : a" {2 4}
    5 " - {d d c} : a" {1 2}
    6 "- {d c b a} : a" {}
    7 "-{\"a\"} : b" {1 2 3}
    8 "- c : a" {1 2 4}
    9 "-c : a"  {1 2 4}
    10 "-\"c\" : a"  {1 2 4}
  } {
  breakpoint
    do_execsql_test 1.$tn {
      SELECT rowid FROM t1($q)