/ Check-in [03c60134]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX flag to custom tokenizer implementations.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 03c601344ed56b316bcc5fd02f6648b0009ba61b
User & Date: dan 2017-01-23 07:06:27
Context
2017-01-23
15:58
Add the missing SQLITE_API symbol to test_delete.c (it is not added automatically as this file is not part of the amalgamation). check-in: 7a4f512d user: dan tags: trunk
07:06
Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX flag to custom tokenizer implementations. check-in: 03c60134 user: dan tags: trunk
2017-01-22
16:34
Fix an uninitialized variable in the command-line shell. check-in: 06b8001a user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_expr.c.

1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
  char *z = 0;

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pAppend;

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
    int n;
    sqlite3Fts5Dequote(z);
    n = (int)strlen(z);
    rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  }
  sqlite3_free(z);
  if( rc || (rc = sCtx.rc) ){







|







1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
  char *z = 0;

  memset(&sCtx, 0, sizeof(TokenCtx));
  sCtx.pPhrase = pAppend;

  rc = fts5ParseStringFromToken(pToken, &z);
  if( rc==SQLITE_OK ){
    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
    int n;
    sqlite3Fts5Dequote(z);
    n = (int)strlen(z);
    rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
  }
  sqlite3_free(z);
  if( rc || (rc = sCtx.rc) ){

Changes to ext/fts5/test/fts5synonym.test.

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
reset_db
fts5_tclnum_register db

foreach {tn expr res} {
  1  {abc}                           {"abc"}
  2  {one}                           {"one"|"i"|"1"}
  3  {3}                             {"3"|"iii"|"three"}
  4  {3*}                            {"3"|"iii"|"three" *}
} {
  do_execsql_test 4.1.$tn {
    SELECT fts5_expr($expr, 'tokenize=tclnum')
  } [list $res]
}

do_execsql_test 4.2.1 {







|







148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
reset_db
fts5_tclnum_register db

foreach {tn expr res} {
  1  {abc}                           {"abc"}
  2  {one}                           {"one"|"i"|"1"}
  3  {3}                             {"3"|"iii"|"three"}
  4  {3*}                            {"3" *}
} {
  do_execsql_test 4.1.$tn {
    SELECT fts5_expr($expr, 'tokenize=tclnum')
  } [list $res]
}

do_execsql_test 4.2.1 {

Changes to ext/fts5/test/fts5tokenizer.test.

257
258
259
260
261
262
263
264







































265
266
  INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
  SELECT term FROM e7;
  ROLLBACK;
} {
  brown dog fox jump lazi over quick the
}








































finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
  INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
  CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
  SELECT term FROM e7;
  ROLLBACK;
} {
  brown dog fox jump lazi over quick the
}

#-------------------------------------------------------------------------
# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
# implementation.
#
reset_db
proc tcl_create {args} { return "tcl_tokenize" }
sqlite3_fts5_create_tokenizer db tcl tcl_create
set ::flags [list]
proc tcl_tokenize {tflags text} {
  lappend ::flags $tflags
  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
    sqlite3_fts5_token $w $iStart $iEnd
  }
}

do_execsql_test 9.1.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
  INSERT INTO t1 VALUES('abc');
  INSERT INTO t1 VALUES('xyz');
} {}
do_test 9.1.2 { set ::flags } {document document}

set ::flags [list]
do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
do_test 9.2.2 { set ::flags } {query}

set ::flags [list]
do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
do_test 9.3.2 { set ::flags } {prefixquery}

set ::flags [list]
do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
do_test 9.4.2 { set ::flags } {prefixquery}

set ::flags [list]
do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
do_test 9.5.2 { set ::flags } {query}


finish_test