SQLite

Check-in [26d2def8a5]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix an FTS3/4 problem with handling empty tokenizer declarations (e.g. "CREATE VIRTUAL TABLE t(tokenize=);").
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 26d2def8a53094356008861636d66f9ae8f2448a
User & Date: dan 2015-03-19 16:25:42.953
Context
2015-03-19
20:09
If a virtual table is created with a malformed UTF8 name in a UTF16 database, make sure that does not cause problems. (check-in: b74cb0a92b user: drh tags: trunk)
18:56
Fix a problem with creating virtual table with names specified using malformed utf-8 within utf-16 databases. (check-in: 9969cff2d0 user: dan tags: malformed-vtab-name)
16:25
Fix an FTS3/4 problem with handling empty tokenizer declarations (e.g. "CREATE VIRTUAL TABLE t(tokenize=);"). (check-in: 26d2def8a5 user: dan tags: trunk)
15:52
Silently ignore any attempt to add a prefix index for prefixes zero bytes in size to an fts3/4 table. Or any prefix index size so large that it overflows a 32-bit signed integer. (check-in: ad4b19d2ac user: dan tags: trunk)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_tokenizer.c.
︙︙
157
158
159
160
161
162
163




164
165
166
167
168
169
170

  sqlite3_tokenizer_module *m;

  zCopy = sqlite3_mprintf("%s", zArg);
  if( !zCopy ) return SQLITE_NOMEM;
  zEnd = &zCopy[strlen(zCopy)];

  z = (char *)sqlite3Fts3NextToken(zCopy, &n);




  z[n] = '\0';
  sqlite3Fts3Dequote(z);

  m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
  if( !m ){
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
    rc = SQLITE_ERROR;








>
>
>
>








157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174

  sqlite3_tokenizer_module *m;

  zCopy = sqlite3_mprintf("%s", zArg);
  if( !zCopy ) return SQLITE_NOMEM;
  zEnd = &zCopy[strlen(zCopy)];

  z = (char *)sqlite3Fts3NextToken(zCopy, &n);
  if( z==0 ){
    assert( n==0 );
    z = zCopy;
  }
  z[n] = '\0';
  sqlite3Fts3Dequote(z);

  m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
  if( !m ){
    *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
    rc = SQLITE_ERROR;
︙︙
Changes to test/fts3atoken.test.
︙︙
182
183
184
185
186
187
188
189
190
191
192













193
194
195


  do_test 5.2 {
    set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
    execsql { INSERT INTO x1 VALUES($str) }
  } {}
}


do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}















finish_test








<



>
>
>
>
>
>
>
>
>
>
>
>
>




182
183
184
185
186
187
188

189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207


  do_test 5.2 {
    set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
    execsql { INSERT INTO x1 VALUES($str) }
  } {}
}


do_test fts3token-internal {
  execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}

#-------------------------------------------------------------------------
# Test empty tokenizer names.
#
do_catchsql_test 6.1.1 {
  CREATE VIRTUAL TABLE t3 USING fts4(tokenize="");
} {1 {unknown tokenizer: }}
do_catchsql_test 6.1.2 {
  CREATE VIRTUAL TABLE t3 USING fts4(tokenize=);
} {1 {unknown tokenizer: }}
do_catchsql_test 6.1.3 {
  CREATE VIRTUAL TABLE t3 USING fts4(tokenize="   ");
} {1 {unknown tokenizer:    }}


finish_test
Changes to test/fts3prefix.test.
︙︙
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276

do_execsql_test 6.4.2 {
  SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]

reset_db
do_execsql_test 6.5.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(prefix="2147483647,2147483648,2147483649");
  CREATE VIRTUAL TABLE t2 USING fts4(prefix="");
  INSERT INTO t1 VALUES('He dressed himself in cycling clothes');
  INSERT INTO t2 VALUES('He dressed himself in cycling clothes');
} {}
do_execsql_test 6.5.2 {
  SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]









|








262
263
264
265
266
267
268
269
270
271
272
273
274
275
276

do_execsql_test 6.4.2 {
  SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]

reset_db
do_execsql_test 6.5.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(prefix="2147483647,2147483648,2147483649");
  CREATE VIRTUAL TABLE t2 USING fts4(prefix=);
  INSERT INTO t1 VALUES('He dressed himself in cycling clothes');
  INSERT INTO t2 VALUES('He dressed himself in cycling clothes');
} {}
do_execsql_test 6.5.2 {
  SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
︙︙