Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Allow FTS tokenizers to choose whether or not to consider the "*" character part of tokens or not. This restores the pre-[e21bf7a2ad] behaviour. Also fix a problem causing FTS to interpret tokens beginning with "*" characters as EOF. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
49dfee7cd1c9ab2901b8a871a6cd00b2 |
| User & Date: | dan 2014-10-09 15:08:17 |
Context
|
2014-10-10
| ||
| 13:08 | Cause the command-line shell to return non-zero if the final SQL statement is incomplete. (check-in: 177fff3b user: drh tags: trunk) | |
| 12:56 | Merge all recent trunk changes. (check-in: abfef254 user: drh tags: sessions) | |
|
2014-10-09
| ||
| 15:08 | Allow FTS tokenizers to choose whether or not to consider the "*" character part of tokens or not. This restores the pre-[e21bf7a2ad] behaviour. Also fix a problem causing FTS to interpret tokens beginning with "*" characters as EOF. (check-in: 49dfee7c user: dan tags: trunk) | |
| 14:10 | Add a test case for the memory leak fixed by the previous check-in. (check-in: bae36d54 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3_expr.c.
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
sqlite3_tokenizer_cursor *pCursor;
Fts3Expr *pRet = 0;
int i = 0;
/* Set variable i to the maximum number of bytes of input to tokenize. */
for(i=0; i<n; i++){
if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
if( z[i]=='*' || z[i]=='"' ) break;
}
*pnConsumed = i;
rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
|
| |
186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 |
sqlite3_tokenizer_cursor *pCursor;
Fts3Expr *pRet = 0;
int i = 0;
/* Set variable i to the maximum number of bytes of input to tokenize. */
for(i=0; i<n; i++){
if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
if( z[i]=='"' ) break;
}
*pnConsumed = i;
rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
if( rc==SQLITE_OK ){
const char *zToken;
int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
|
Changes to test/fts3expr4.test.
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
..
48
49
50
51
52
53
54
55
56
57
|
ifcapable !fts3||!icu {
finish_test
return
}
set sqlite_fts3_enable_parentheses 1
proc test_icu_fts3expr {expr} {
db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')}
}
proc do_icu_expr_test {tn expr res} {
uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res]
}
#-------------------------------------------------------------------------
#
do_icu_expr_test 1.1 "abcd" {PHRASE 3 0 abcd}
do_icu_expr_test 1.2 " tag " {PHRASE 3 0 tag}
do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z}
................................................................................
do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word}
set sqlite_fts3_enable_parentheses 0
do_icu_expr_test 2.1 {
f (e NEAR/2 a)
} {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}
finish_test
|
|
|
|
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
|
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
..
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
ifcapable !fts3||!icu {
finish_test
return
}
set sqlite_fts3_enable_parentheses 1
proc test_fts3expr {tokenizer expr} {
db one {SELECT fts3_exprtest($tokenizer, $expr, 'a', 'b', 'c')}
}
proc do_icu_expr_test {tn expr res} {
uplevel [list do_test $tn [list test_fts3expr icu $expr] [list {*}$res]]
}
proc do_simple_expr_test {tn expr res} {
uplevel [list do_test $tn [list test_fts3expr simple $expr] [list {*}$res]]
}
#-------------------------------------------------------------------------
#
do_icu_expr_test 1.1 "abcd" {PHRASE 3 0 abcd}
do_icu_expr_test 1.2 " tag " {PHRASE 3 0 tag}
do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z}
................................................................................
do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word}
set sqlite_fts3_enable_parentheses 0
do_icu_expr_test 2.1 {
f (e NEAR/2 a)
} {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}
#-------------------------------------------------------------------------
#
do_simple_expr_test 3.1 {*lOl* *h4h*} {
AND {PHRASE 3 0 lol+} {PHRASE 3 0 h4h+}
}
do_icu_expr_test 3.2 {*lOl* *h4h*} {
AND {AND {AND {PHRASE 3 0 *} {PHRASE 3 0 lol+}} {PHRASE 3 0 *}} {PHRASE 3 0 h4h+}
}
do_simple_expr_test 3.3 { * } { }
do_simple_expr_test 3.4 { *a } { PHRASE 3 0 a }
do_simple_expr_test 3.5 { a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
do_simple_expr_test 3.6 { *a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
do_simple_expr_test 3.7 { *"abc" } { PHRASE 3 0 abc }
do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc }
do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c }
do_icu_expr_test 3.9 { "ab*c" } { PHRASE 3 0 ab+ * c }
do_icu_expr_test 3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}}
finish_test
|