SQLite

Check-in [49dfee7cd1]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Allow FTS tokenizers to choose whether or not to consider the "*" character part of tokens or not. This restores the pre-[e21bf7a2ad] behaviour. Also fix a problem causing FTS to interpret tokens beginning with "*" characters as EOF.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 49dfee7cd1c9ab2901b8a871a6cd00b2ead76801
User & Date: dan 2014-10-09 15:08:17.615
Context
2014-10-10
13:08
Cause the command-line shell to return non-zero if the final SQL statement is incomplete. (check-in: 177fff3b98 user: drh tags: trunk)
12:56
Merge all recent trunk changes. (check-in: abfef25472 user: drh tags: sessions)
2014-10-09
15:08
Allow FTS tokenizers to choose whether or not to consider the "*" character part of tokens or not. This restores the pre-[e21bf7a2ad] behaviour. Also fix a problem causing FTS to interpret tokens beginning with "*" characters as EOF. (check-in: 49dfee7cd1 user: dan tags: trunk)
14:10
Add a test case for the memory leak fixed by the previous check-in. (check-in: bae36d5446 user: drh tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_expr.c.
︙︙
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200

  sqlite3_tokenizer_cursor *pCursor;
  Fts3Expr *pRet = 0;
  int i = 0;

  /* Set variable i to the maximum number of bytes of input to tokenize. */
  for(i=0; i<n; i++){
    if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
    if( z[i]=='*' || z[i]=='"' ) break;
  }

  *pnConsumed = i;
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;








|








186
187
188
189
190
191
192
193
194
195
196
197
198
199
200

  sqlite3_tokenizer_cursor *pCursor;
  Fts3Expr *pRet = 0;
  int i = 0;

  /* Set variable i to the maximum number of bytes of input to tokenize. */
  for(i=0; i<n; i++){
    if( sqlite3_fts3_enable_parentheses && (z[i]=='(' || z[i]==')') ) break;
    if( z[i]=='"' ) break;
  }

  *pnConsumed = i;
  rc = sqlite3Fts3OpenTokenizer(pTokenizer, pParse->iLangid, z, i, &pCursor);
  if( rc==SQLITE_OK ){
    const char *zToken;
    int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0;
︙︙
Changes to test/fts3expr4.test.
︙︙
20
21
22
23
24
25
26
27
28
29
30
31
32




33
34
35
36
37
38
39

ifcapable !fts3||!icu {
  finish_test
  return
}

set sqlite_fts3_enable_parentheses 1

proc test_icu_fts3expr {expr} {
  db one {SELECT fts3_exprtest('icu', $expr, 'a', 'b', 'c')}
}

proc do_icu_expr_test {tn expr res} {
  uplevel [list do_test $tn [list test_icu_fts3expr $expr] $res]




}

#-------------------------------------------------------------------------
#
do_icu_expr_test 1.1 "abcd"    {PHRASE 3 0 abcd}
do_icu_expr_test 1.2 " tag "   {PHRASE 3 0 tag}
do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z}








|
|



|
>
>
>
>








20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

ifcapable !fts3||!icu {
  finish_test
  return
}

set sqlite_fts3_enable_parentheses 1

proc test_fts3expr {tokenizer expr} {
  db one {SELECT fts3_exprtest($tokenizer, $expr, 'a', 'b', 'c')}
}

proc do_icu_expr_test {tn expr res} {
  uplevel [list do_test $tn [list test_fts3expr icu $expr] [list {*}$res]]
}

proc do_simple_expr_test {tn expr res} {
  uplevel [list do_test $tn [list test_fts3expr simple $expr] [list {*}$res]]
}

#-------------------------------------------------------------------------
#
do_icu_expr_test 1.1 "abcd"    {PHRASE 3 0 abcd}
do_icu_expr_test 1.2 " tag "   {PHRASE 3 0 tag}
do_icu_expr_test 1.3 {"x y z"} {PHRASE 3 0 x y z}
︙︙
48
49
50
51
52
53
54





















55
56
57

do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word}

set sqlite_fts3_enable_parentheses 0

do_icu_expr_test 2.1 {
  f (e NEAR/2 a)
} {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}






















finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>




52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

do_icu_expr_test 1.8 {d:word} {PHRASE 3 0 d:word}

set sqlite_fts3_enable_parentheses 0

do_icu_expr_test 2.1 {
  f (e NEAR/2 a)
} {AND {AND {AND {PHRASE 3 0 f} {PHRASE 3 0 (}} {NEAR/2 {PHRASE 3 0 e} {PHRASE 3 0 a}}} {PHRASE 3 0 )}}

#-------------------------------------------------------------------------
#
do_simple_expr_test 3.1 {*lOl* *h4h*} {
  AND {PHRASE 3 0 lol+} {PHRASE 3 0 h4h+}
}

do_icu_expr_test 3.2 {*lOl* *h4h*} {
  AND {AND {AND {PHRASE 3 0 *} {PHRASE 3 0 lol+}} {PHRASE 3 0 *}} {PHRASE 3 0 h4h+}
}

do_simple_expr_test 3.3 { * }    { }
do_simple_expr_test 3.4 { *a }   { PHRASE 3 0 a }
do_simple_expr_test 3.5 { a*b }  { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
do_simple_expr_test 3.6 { *a*b } { AND {PHRASE 3 0 a+} {PHRASE 3 0 b} }
do_simple_expr_test 3.7 { *"abc" } { PHRASE 3 0 abc }
do_simple_expr_test 3.8 { "abc"* } { PHRASE 3 0 abc }
do_simple_expr_test 3.8 { "ab*c" } { PHRASE 3 0 ab+ c }

do_icu_expr_test    3.9 { "ab*c" } { PHRASE 3 0 ab+ * c }
do_icu_expr_test    3.10 { ab*c } { AND {PHRASE 3 0 ab+} {PHRASE 3 0 c}}

finish_test