Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix an FTS3/4 problem with handling empty tokenizer declarations (e.g. "CREATE VIRTUAL TABLE t(tokenize=);"). |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
26d2def8a53094356008861636d66f9a |
| User & Date: | dan 2015-03-19 16:25:42 |
Context
|
2015-03-19
| ||
| 20:09 | If a virtual table is created with a malformed UTF8 name in a UTF16 database, make sure that does not cause problems. check-in: b74cb0a9 user: drh tags: trunk | |
| 18:56 | Fix a problem with creating virtual table with names specified using malformed utf-8 within utf-16 databases. check-in: 9969cff2 user: dan tags: malformed-vtab-name | |
| 16:25 | Fix an FTS3/4 problem with handling empty tokenizer declarations (e.g. "CREATE VIRTUAL TABLE t(tokenize=);"). check-in: 26d2def8 user: dan tags: trunk | |
| 15:52 | Silently ignore any attempt to add a prefix index for prefixes zero bytes in size to an fts3/4 table. Or any prefix index size so large that it overflows a 32-bit signed integer. check-in: ad4b19d2 user: dan tags: trunk | |
Changes
Changes to ext/fts3/fts3_tokenizer.c.
157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
sqlite3_tokenizer_module *m;
zCopy = sqlite3_mprintf("%s", zArg);
if( !zCopy ) return SQLITE_NOMEM;
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
z[n] = '\0';
sqlite3Fts3Dequote(z);
m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
|
> > > > |
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 |
sqlite3_tokenizer_module *m;
zCopy = sqlite3_mprintf("%s", zArg);
if( !zCopy ) return SQLITE_NOMEM;
zEnd = &zCopy[strlen(zCopy)];
z = (char *)sqlite3Fts3NextToken(zCopy, &n);
if( z==0 ){
assert( n==0 );
z = zCopy;
}
z[n] = '\0';
sqlite3Fts3Dequote(z);
m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1);
if( !m ){
*pzErr = sqlite3_mprintf("unknown tokenizer: %s", z);
rc = SQLITE_ERROR;
|
Changes to test/fts3atoken.test.
182 183 184 185 186 187 188 189 190 191 192 193 194 195 |
do_test 5.2 {
set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
execsql { INSERT INTO x1 VALUES($str) }
} {}
}
do_test fts3token-internal {
execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}
finish_test
|
< > > > > > > > > > > > > > |
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 |
do_test 5.2 {
set str [cp_to_str {19968 26085 32822 32645 27874 23433 20986}]
execsql { INSERT INTO x1 VALUES($str) }
} {}
}
do_test fts3token-internal {
execsql { SELECT fts3_tokenizer_internal_test() }
} {ok}
#-------------------------------------------------------------------------
# Test empty tokenizer names.
#
do_catchsql_test 6.1.1 {
CREATE VIRTUAL TABLE t3 USING fts4(tokenize="");
} {1 {unknown tokenizer: }}
do_catchsql_test 6.1.2 {
CREATE VIRTUAL TABLE t3 USING fts4(tokenize=);
} {1 {unknown tokenizer: }}
do_catchsql_test 6.1.3 {
CREATE VIRTUAL TABLE t3 USING fts4(tokenize=" ");
} {1 {unknown tokenizer: }}
finish_test
|
Changes to test/fts3prefix.test.
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 |
do_execsql_test 6.4.2 {
SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
reset_db
do_execsql_test 6.5.1 {
CREATE VIRTUAL TABLE t1 USING fts4(prefix="2147483647,2147483648,2147483649");
CREATE VIRTUAL TABLE t2 USING fts4(prefix="");
INSERT INTO t1 VALUES('He dressed himself in cycling clothes');
INSERT INTO t2 VALUES('He dressed himself in cycling clothes');
} {}
do_execsql_test 6.5.2 {
SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
finish_test
|
| |
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 |
do_execsql_test 6.4.2 {
SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
reset_db
do_execsql_test 6.5.1 {
CREATE VIRTUAL TABLE t1 USING fts4(prefix="2147483647,2147483648,2147483649");
CREATE VIRTUAL TABLE t2 USING fts4(prefix=);
INSERT INTO t1 VALUES('He dressed himself in cycling clothes');
INSERT INTO t2 VALUES('He dressed himself in cycling clothes');
} {}
do_execsql_test 6.5.2 {
SELECT md5sum(quote(root)) FROM t1_segdir;
} [db eval {SELECT md5sum(quote(root)) FROM t2_segdir}]
finish_test
|