/ Check-in [11eb8e87]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix the fts5 "prefix=" option to match the documentation (space separated list, multiple prefix= options supported). The undocumented comma-separated format (compatible with fts4) still works.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 11eb8e877e2ba859ef6b44318f286597186dfaf2
User & Date: dan 2015-11-25 11:56:24
Context
2015-11-25
14:00
Simplification of the error code translator in os_unix.c. Code cleanup only. The logic is unchanged. check-in: 2a20f793 user: drh tags: trunk
11:56
Fix the fts5 "prefix=" option to match the documentation (space separated list, multiple prefix= options supported). The undocumented comma-separated format (compatible with fts4) still works. check-in: 11eb8e87 user: dan tags: trunk
01:57
Enhancement the virtual table interface to support LIKE, GLOB, and REGEXP operators. Also add the sqlite3_strlike() interface, which might be useful as part of the implementation of LIKE on some virtual tables. check-in: a6bfd469 user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_config.c.

212
213
214
215
216
217
218

219
220
221
222
223

224

225
226
227

228




















229
230
231
232
233
234
235
236
237
238
239
240
241
242

243

244
245

246

247
248
249
250
251
252
253
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;
  int nCmd = (int)strlen(zCmd);
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    const char *p;

    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);

    }

    p = zArg;
    while( rc==SQLITE_OK && p[0] ){
      int nPre = 0;

      while( p[0]==' ' ) p++;




















      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
        nPre = nPre*10 + (p[0] - '0');
        p++;
      }
      while( p[0]==' ' ) p++;
      if( p[0]==',' ){
        p++;
      }else if( p[0] ){
        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
        rc = SQLITE_ERROR;
      }
      if( rc==SQLITE_OK && (nPre==0 || nPre>=1000) ){
        *pzErr = sqlite3_mprintf("prefix length out of range: %d", nPre);
        rc = SQLITE_ERROR;

      }

      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;

    }

    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
    int nArg = (int)strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);







>
|
<
<
<

>

>

|

>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




<
<
<
<
<
<
|
|
|

>

>


>

>







212
213
214
215
216
217
218
219
220



221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253






254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;
  int nCmd = (int)strlen(zCmd);
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    const char *p;
    int bFirst = 1;
    if( pConfig->aPrefix==0 ){



      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
      if( rc ) return rc;
    }

    p = zArg;
    while( 1 ){
      int nPre = 0;

      while( p[0]==' ' ) p++;
      if( bFirst==0 && p[0]==',' ){
        p++;
        while( p[0]==' ' ) p++;
      }else if( p[0]=='\0' ){
        break;
      }
      if( p[0]<'0' || p[0]>'9' ){
        *pzErr = sqlite3_mprintf("malformed prefix=... directive");
        rc = SQLITE_ERROR;
        break;
      }

      if( pConfig->nPrefix==FTS5_MAX_PREFIX_INDEXES ){
        *pzErr = sqlite3_mprintf(
            "too many prefix indexes (max %d)", FTS5_MAX_PREFIX_INDEXES
        );
        rc = SQLITE_ERROR;
        break;
      }

      while( p[0]>='0' && p[0]<='9' && nPre<1000 ){
        nPre = nPre*10 + (p[0] - '0');
        p++;
      }







      if( rc==SQLITE_OK && (nPre<=0 || nPre>=1000) ){
        *pzErr = sqlite3_mprintf("prefix length out of range (max 999)");
        rc = SQLITE_ERROR;
        break;
      }

      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
      bFirst = 0;
    }
    assert( pConfig->nPrefix<=FTS5_MAX_PREFIX_INDEXES );
    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
    int nArg = (int)strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);

Changes to ext/fts5/test/fts5config.test.

38
39
40
41
42
43
44


45
46
47
48
49
50
51
...
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132

133
134
135
136
137
138
139
...
199
200
201
202
203
204
205











206
207
208
#-------------------------------------------------------------------------
# Syntax errors in the prefix= option.
#
foreach {tn opt} {
  1 {prefix=x}  
  2 {prefix='x'}
  3 {prefix='$'}


} {
  set res [list 1 {malformed prefix=... directive}]
  do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}

#-------------------------------------------------------------------------
# Syntax errors in the 'rank' option.
................................................................................
do_catchsql_test 5.3 {
  CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
} {1 {unrecognized token: "]"}}

#-------------------------------------------------------------------------
# Errors in prefix= directives.
#
do_catchsql_test 6.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix=1, prefix=2);
} {1 {multiple prefix=... directives}}
do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
} {1 {prefix length out of range: 1001}}
do_catchsql_test 6.3 {
  CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
} {1 {prefix length out of range: 0}}
do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1  , 1000000');
} {1 {malformed prefix=... directive}}


#-------------------------------------------------------------------------
# Duplicate tokenize= and other options.
#
do_catchsql_test 7.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii);
} {1 {multiple tokenize=... directives}}
................................................................................
do_execsql_test 9.3.4 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
} {}

do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
} {1 {SQL logic error or missing database}}












finish_test








>
>







 







<
<
<


|


|


<
>







 







>
>
>
>
>
>
>
>
>
>
>



38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
...
116
117
118
119
120
121
122



123
124
125
126
127
128
129
130

131
132
133
134
135
136
137
138
...
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
#-------------------------------------------------------------------------
# Syntax errors in the prefix= option.
#
foreach {tn opt} {
  1 {prefix=x}  
  2 {prefix='x'}
  3 {prefix='$'}
  4 {prefix='1,2,'}
  5 {prefix=',1'}
} {
  set res [list 1 {malformed prefix=... directive}]
  do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}

#-------------------------------------------------------------------------
# Syntax errors in the 'rank' option.
................................................................................
do_catchsql_test 5.3 {
  CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
} {1 {unrecognized token: "]"}}

#-------------------------------------------------------------------------
# Errors in prefix= directives.
#



do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
} {1 {prefix length out of range (max 999)}}
do_catchsql_test 6.3 {
  CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
} {1 {prefix length out of range (max 999)}}
do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1  , 1000000');

} {1 {prefix length out of range (max 999)}}

#-------------------------------------------------------------------------
# Duplicate tokenize= and other options.
#
do_catchsql_test 7.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii);
} {1 {multiple tokenize=... directives}}
................................................................................
do_execsql_test 9.3.4 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
} {}

do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
} {1 {SQL logic error or missing database}}

#-------------------------------------------------------------------------
# Too many prefix indexes. Maximum allowed is 31.
#
foreach {tn spec} {
  1 {prefix="1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
  2 {prefix="1 2 3 4", prefix="5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32"}
} {
  set sql "CREATE VIRTUAL TABLE xyz USING fts5(x, $spec)"
  do_catchsql_test 10.$tn $sql {1 {too many prefix indexes (max 31)}}
}

finish_test

Changes to ext/fts5/test/fts5prefix.test.

293
294
295
296
297
298
299
300










































301
302
303
  5 x {xb*}
  6 x {xc*}
} {
  set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"]
  set query "$col : $pattern"
  do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res
}











































finish_test










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
  5 x {xb*}
  6 x {xc*}
} {
  set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"]
  set query "$col : $pattern"
  do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res
}

#-------------------------------------------------------------------------
# Check that the various ways of creating prefix indexes produce the
# same database on disk.
#
save_prng_state
foreach {tn create} {
  1 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1,2,3") }
  2 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2 3") }
  3 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix=1, prefix=2, prefix=3) }
  4 { CREATE VIRTUAL TABLE tt USING fts5(x, y, prefix="1 2", prefix=3) }
} {
  execsql { DROP TABLE IF EXISTS tt }
  restore_prng_state
  execsql $create
  execsql {
    INSERT INTO tt VALUES('cc b ggg ccc aa eee hh', 'aa g b hh a e');
    INSERT INTO tt VALUES('cc bb cc gg j g cc', 'ii jjj ggg jjj cc cc');
    INSERT INTO tt VALUES('h eee cc h iii', 'aaa iii dd iii dd');
    INSERT INTO tt VALUES('jjj hh eee c e b gg', 'j bbb jj ddd jj');
    INSERT INTO tt VALUES('ii hhh aaa ff c hhh iii', 'j cc hh bb e');
    INSERT INTO tt VALUES('e fff hhh i aaa', 'g b aa gg c aa dd');
    INSERT INTO tt VALUES('i aaa ccc gg hhh aa h', 'j bbb bbb d ff');
    INSERT INTO tt VALUES('g f gg ff ff jjj d', 'jjj d j fff fff ee j');
    INSERT INTO tt VALUES('a cc e ccc jjj c', 'ccc iii d bb a eee g');
    INSERT INTO tt VALUES('jj hh hh bb bbb gg', 'j c jjj bb iii f');
    INSERT INTO tt VALUES('a ggg g cc ccc aa', 'jjj j j aaa c');
    INSERT INTO tt VALUES('ddd j dd b i', 'aaa bbb iii ggg ff ccc ddd');
    INSERT INTO tt VALUES('jj ii hh c ii h gg', 'hhh bbb ddd bbb hh g ggg');
    INSERT INTO tt VALUES('aa hhh ccc h ggg ccc', 'iii d jj a ff ii');
  }

  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM tt_data} {puts $r}

  if {$tn==1} {
    set ::checksum [execsql {SELECT md5sum(id, block) FROM tt_data}]
  } else {
    do_execsql_test 7.$tn {
      SELECT md5sum(id, block) FROM tt_data
    } [list $::checksum]
  }
}

finish_test