Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX flag to custom tokenizer implementations. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
03c601344ed56b316bcc5fd02f6648b0 |
| User & Date: | dan 2017-01-23 07:06:27 |
Context
|
2017-01-23
| ||
| 15:58 | Add the missing SQLITE_API symbol to test_delete.c (it is not added automatically as this file is not part of the amalgamation). check-in: 7a4f512d user: dan tags: trunk | |
| 07:06 | Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX flag to custom tokenizer implementations. check-in: 03c60134 user: dan tags: trunk | |
|
2017-01-22
| ||
| 16:34 | Fix an uninitialized variable in the command-line shell. check-in: 06b8001a user: drh tags: trunk | |
Changes
Changes to ext/fts5/fts5_expr.c.
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 |
char *z = 0;
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pAppend;
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
int n;
sqlite3Fts5Dequote(z);
n = (int)strlen(z);
rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
}
sqlite3_free(z);
if( rc || (rc = sCtx.rc) ){
|
| |
1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 |
char *z = 0;
memset(&sCtx, 0, sizeof(TokenCtx));
sCtx.pPhrase = pAppend;
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
int n;
sqlite3Fts5Dequote(z);
n = (int)strlen(z);
rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
}
sqlite3_free(z);
if( rc || (rc = sCtx.rc) ){
|
Changes to ext/fts5/test/fts5synonym.test.
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
reset_db
fts5_tclnum_register db
foreach {tn expr res} {
1 {abc} {"abc"}
2 {one} {"one"|"i"|"1"}
3 {3} {"3"|"iii"|"three"}
4 {3*} {"3"|"iii"|"three" *}
} {
do_execsql_test 4.1.$tn {
SELECT fts5_expr($expr, 'tokenize=tclnum')
} [list $res]
}
do_execsql_test 4.2.1 {
|
| |
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
reset_db
fts5_tclnum_register db
foreach {tn expr res} {
1 {abc} {"abc"}
2 {one} {"one"|"i"|"1"}
3 {3} {"3"|"iii"|"three"}
4 {3*} {"3" *}
} {
do_execsql_test 4.1.$tn {
SELECT fts5_expr($expr, 'tokenize=tclnum')
} [list $res]
}
do_execsql_test 4.2.1 {
|
Changes to ext/fts5/test/fts5tokenizer.test.
257 258 259 260 261 262 263 264 265 266 |
INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
SELECT term FROM e7;
ROLLBACK;
} {
brown dog fox jump lazi over quick the
}
finish_test
|
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 |
INSERT INTO e6 VALUES('theAquickBbrownCfoxDjumpedWoverXtheYlazyZdog');
CREATE VIRTUAL TABLE e7 USING fts5vocab(e6, 'row');
SELECT term FROM e7;
ROLLBACK;
} {
brown dog fox jump lazi over quick the
}
#-------------------------------------------------------------------------
# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
# implementation.
#
reset_db
proc tcl_create {args} { return "tcl_tokenize" }
sqlite3_fts5_create_tokenizer db tcl tcl_create
set ::flags [list]
proc tcl_tokenize {tflags text} {
lappend ::flags $tflags
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
}
}
do_execsql_test 9.1.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
INSERT INTO t1 VALUES('abc');
INSERT INTO t1 VALUES('xyz');
} {}
do_test 9.1.2 { set ::flags } {document document}
set ::flags [list]
do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
do_test 9.2.2 { set ::flags } {query}
set ::flags [list]
do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
do_test 9.3.2 { set ::flags } {prefixquery}
set ::flags [list]
do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
do_test 9.4.2 { set ::flags } {prefixquery}
set ::flags [list]
do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
do_test 9.5.2 { set ::flags } {query}
finish_test
|