Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts3-refactor Excluding Merge-Ins
This is equivalent to a diff from 4bd43307 to 7a46d1eb
2009-11-19
| ||
15:26 | Merge fts3-refactor into the main trunk. (check-in: 551dbe30 user: dan tags: trunk) | |
15:25 | Fix some fts3 related issues with the autoconf and amalgamation build systems. (Closed-Leaf check-in: 3b179247 user: dan) | |
14:57 | Merge with [4bd4330709]. (check-in: 7a46d1eb user: dan tags: fts3-refactor) | |
14:52 | Merge the fts3-refactor branch with the trunk. (check-in: c8d2bd37 user: dan tags: fts3-refactor) | |
14:48 | Fix a bug introduced with recent optimizations: The unary minus operator is TK_UMINUS, not TK_MINUS. (check-in: 4bd43307 user: drh tags: trunk) | |
2009-11-18
| ||
23:01 | Performance improvement by avoiding unnecessary calls to memset(). (check-in: 85940468 user: drh tags: trunk) | |
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
274 275 276 277 278 279 280 281 282 | #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif #include <assert.h> #include <stdlib.h> | > > | | | < < < > | 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 | #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) # define SQLITE_CORE 1 #endif #include "fts3Int.h" #include <assert.h> #include <stdlib.h> #include <stddef.h> #include <stdio.h> #include <string.h> #include "fts3.h" #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif /* TODO(shess) MAN, this thing needs some refactoring. At minimum, it ** would be nice to order the file better, perhaps something along the ** lines of: ** ** - utility functions |
︙ | ︙ | |||
309 310 311 312 313 314 315 | #if 0 # define FTSTRACE(A) printf A; fflush(stdout) #else # define FTSTRACE(A) #endif | < < < < < < < < < < < < < < < < < < < | 309 310 311 312 313 314 315 316 317 318 319 320 321 322 | #if 0 # define FTSTRACE(A) printf A; fflush(stdout) #else # define FTSTRACE(A) #endif typedef enum DocListType { DL_DOCIDS, /* docids only */ DL_POSITIONS, /* docids + positions */ DL_POSITIONS_OFFSETS /* docids + positions + offsets */ } DocListType; /* |
︙ | ︙ | |||
353 354 355 356 357 358 359 | enum { POS_END = 0, /* end of this position list */ POS_COLUMN, /* followed by new column number */ POS_BASE }; | < < < < < < < | | | | > | | > | | | > | | > > > > | | < < < < < < < < < < < < < | < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | | 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 | enum { POS_END = 0, /* end of this position list */ POS_COLUMN, /* followed by new column number */ POS_BASE }; /* utility functions */ /* CLEAR() and SCRAMBLE() abstract memset() on a pointer to a single ** record to prevent errors of the form: ** ** my_function(SomeType *b){ ** memset(b, '\0', sizeof(b)); // sizeof(b)!=sizeof(*b) ** } */ /* TODO(shess) Obvious candidates for a header file. */ #define CLEAR(b) memset(b, '\0', sizeof(*(b))) #ifndef NDEBUG # define SCRAMBLE(b) memset(b, 0x55, sizeof(*(b))) #else # define SCRAMBLE(b) #endif /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. ** The number of bytes written is returned. */ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; sqlite_uint64 vu = v; do{ *q++ = (unsigned char) ((vu & 0x7f) | 0x80); vu >>= 7; }while( vu!=0 ); q[-1] &= 0x7f; /* turn off high bit in final byte */ assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); return (int) (q - (unsigned char *)p); } /* ** Read a 64-bit variable-length integer from memory starting at p[0]. ** Return the number of bytes read, or 0 on error. ** The value is stored in *v. */ int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; while( (*q & 0x80) == 0x80 ){ x += y * (*q++ & 0x7f); y <<= 7; if( q - (unsigned char *)p >= FTS3_VARINT_MAX ){ /* bad data */ assert( 0 ); return 0; } } x += y * (*q++); *v = (sqlite_int64) x; return (int) (q - (unsigned char *)p); } /* ** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a ** 32-bit integer before it is returned. */ int sqlite3Fts3GetVarint32(const char *p, int *pi){ sqlite_int64 i; int ret = sqlite3Fts3GetVarint(p, &i); *pi = (int) i; assert( *pi==i ); return ret; } /* ** Return the number of bytes required to store the value passed as the ** first argument in varint form. */ int sqlite3Fts3VarintLen(sqlite3_uint64 v){ int i = 0; do{ i++; v >>= 7; }while( v!=0 ); return i; } /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the ** input does not begin with a quote character, then this routine ** is a no-op. ** ** Examples: ** ** "abc" becomes abc ** 'xyz' becomes xyz ** [pqr] becomes pqr ** `mno` becomes mno */ void sqlite3Fts3Dequote(char *z){ int quote; int i, j; quote = z[0]; switch( quote ){ case '\'': break; case '"': break; case '`': break; /* For MySQL compatibility */ case '[': quote = ']'; break; /* For MS SqlServer compatibility */ default: return; |
︙ | ︙ | |||
2633 2634 2635 2636 2637 2638 2639 | } }else{ z[j++] = z[i]; } } } | < < < < < < < < < < < < < < < | | < < < | < < < < < < | | | | | < < < < < < < < < < < < < | | < < | < | | < | < < < < < < < < < < < < < < < < | < < < < < < > < < > > | | < < < > | | < < < < < < < | | < < < < < < < > | | < > | < < < | < < < < < < < < < < < < < < < | < < < < < > | < | < > | < < < < < < < < < | < < < | < < | | | | < < < < < < < < | < < | | < < | < < < < < | < | | | < < < < | < < | < | < < < < < < < < < > | < < < < < < < < < | < < < < < < < | < < < > | < < < < < < | < < < | < | < < | < | < < < < < < < < < < | < < | | < < < < < | < < < > > | < | < < < < < < | < < | < < < < | < < | < | < < | | < | | | < < < < | | | < < | < | < < < | < < < | < | < < < < < < | < < < < | | < < < < < < < < > | > > | < < < < < < < < < < | < < < < < | | | < > | | | < | < | < > | < > > < < < | < < | < | < < | < < | < < | < < < < | < < | > > > > | > > > > > > > > | | < < < > > > | | > | < < < < < < < < | < | < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | > < | < < | > > | | < < < < | < | < > > | < < < < < < < < < < < < | | > > | | < < < < | < < | < | < < < < | | < | < < < | < > | < | < | < < < < < < | < < < | < < < < < < < < < < | < < < > > > > > > | < < < | < | < | < < < < < < < < < < < | < | | < < | | > > | < < < < < < > < < | > > > > > > > > > > | > | | < > > > > > | < < | < < < < < < < < < < < < < < < < < < < < | | < > | < > | | | | > > | | < > | > > > > > > > | | < < < < < < < < < | | < < < < | < < < < < < | | < < < < | < < < < < < < < < < < < < < < < < < < < | | < > | < < < | < > | < < | < < | | | < < | < < < < < < < | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < > > > > > > > > | < | | < > > | | > | | > > > > > | < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < | < < < > > < < < > > > > > > > < < < < < < < < < < < | < < < < < > > | < < < < < < < > > > | > | < < < < | < < < | | < < < | < < | < < < < < < < < > | > > | < < | < < < < < > < < < < | | < > > > > > | | < < | | > | < | < < < < < < | | < < < < < < < < < < < < < < < | | > | | < < < < < > | < < < < < < < | < < < < > | > > > | | | | < > | < < < < | < | < | < | < < < < < | | < < < < | < < < > > > > | < | | > > > > | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > | > > | > > > > > | > > > > > > > > > > > > | > > > > > | > > > > > > > > > > > > > | > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > | < < < > > | < < | > | < > | < < | | | | > > > > > > > > > > | | > > > > > > > > > > > > > > > > > > > > > > > | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > | > > > > > > > > > > > > > | > > > > > > | > > > > > > > > > > > > > > > > > | > > | > > > > > | | > | > > > > > | > > > | | > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | < > > > | > > > | > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > | > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | | | | > > | > < < | < | < < | | < | > > > > > | > | > | | > | > > > > > > | < > > | | | | < | | < | | | > > > > > | | | > | | | < | | < < | < < | | | > > < | | < > > > > > | | > | | < | > > > > < < | < | | < | < | < < > < < < < < < < < < < < < < < < < < | | > > > > | < < < | < | | | | > > | | > > > | < > > | < | < | < < < < | | < | < < < < < | | > | | | | | | | | | > | > | < < | | < | < < < < < < < < | > > > > > > > | | > > > | > | < > | > | < < < < < < < < < | < > | < > > | > | < > > < | | < < < | | < | | < < < < < | | < < < < > | | | | | > | | < | < | | | > | | | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < | < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < | < < < < | < | | < < < < < | < < < | < < < < | < < < < < < < | < < | < | < < < < > | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < | < < < < < < < | < < < < < < | < < < < < < | < < < < < < < < | < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < | < < < < < < | < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < | < < < < < < < | < < < < < < < < < < < < | < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < | < < | < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < < < < < < < < < < < < < < | < < < < < < < < | < < < < < | < < < < | < < < < < < < < < | < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < | < < < | < < < < < < < < < < < | < < < < < < < < < < | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < | < < | < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < | < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < | < < | < < < < < < < < < < < < < < < < < < < < < < < < | < < < < | < < < < < < < | < < < < < < < < < < < | < < < < < < < < < | < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < | | | < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < < < < | | | | | > | | > | < | | | < < | < < < | | | | < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < < < < < < < < | < < < < < < < | < < < < < < < < < | < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < | > | | < | < < < < < < < | < < < < | > < | < < | > | < < < > | | < < < < < | < < < < < < < | < < < < < < < < < < < < < < < < < | < < | < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | | < < < < < < < < < < < < | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < | < < < < < | < < | | < < < < < < < < < < < < < < < < < < < < < | < < | < < < < < | | < | < < < < < < < < < < | < < < < < < < < < < < | < < < < < | | | | | | > | > > | < < | < | > > | < < < < < < | < | | < < < < < | > > > | | | | | | > > | | | | | | | | | | | | | | | | | | > > > > > | < < | | | 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 | } }else{ z[j++] = z[i]; } } } static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ sqlite3_int64 iVal; *pp += sqlite3Fts3GetVarint(*pp, &iVal); *pVal += iVal; } static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ if( *pp>=pEnd ){ *pp = 0; }else{ fts3GetDeltaVarint(pp, pVal); } } /* ** The Fts3Cursor.eType member is always set to one of the following. */ #define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ static Fts3Table *cursor_vtab(Fts3Cursor *c){ return (Fts3Table *) c->base.pVtab; } /* ** The xDisconnect() virtual table method. */ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table *)pVtab; int i; assert( p->nPendingData==0 ); /* Free any prepared statements held */ for(i=0; i<SizeofArray(p->aStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSelectLeaves); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); return SQLITE_OK; } /* ** The xDestroy() virtual table method. */ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ int rc; /* Return code */ Fts3Table *p = (Fts3Table *)pVtab; /* Create a script to drop the underlying three storage tables. */ char *zSql = sqlite3_mprintf( "DROP TABLE IF EXISTS %Q.'%q_content';" "DROP TABLE IF EXISTS %Q.'%q_segments';" "DROP TABLE IF EXISTS %Q.'%q_segdir';", p->zDb, p->zName, p->zDb, p->zName, p->zDb, p->zName ); /* If malloc has failed, set rc to SQLITE_NOMEM. Otherwise, try to ** execute the SQL script created above. */ if( zSql ){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); }else{ rc = SQLITE_NOMEM; } /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. ** Otherwise, return an SQLite error code. */ return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); } /* ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. */ static int fts3DeclareVtab(Fts3Table *p){ int i; /* Iterator variable */ int rc; /* Return code */ char *zSql; /* SQL statement passed to declare_vtab() */ char *zCols; /* List of user defined columns */ /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); for(i=1; zCols && i<p->nColumn; i++){ zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName ); if( !zCols || !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_declare_vtab(p->db, zSql); } sqlite3_free(zSql); sqlite3_free(zCols); return rc; } /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done ** as part of the vtab xCreate() method. */ static int fts3CreateTables(Fts3Table *p){ int rc; /* Return code */ int i; /* Iterator variable */ char *zContentCols; /* Columns of %_content table */ char *zSql; /* SQL script to create required tables */ /* Create a list of user columns for the content table */ zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); for(i=0; zContentCols && i<p->nColumn; i++){ char *z = p->azColumn[i]; zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); } /* Create the whole SQL script */ zSql = sqlite3_mprintf( "CREATE TABLE %Q.'%q_content'(%s);" "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);" "CREATE TABLE %Q.'%q_segdir'(" "level INTEGER," "idx INTEGER," "start_block INTEGER," "leaves_end_block INTEGER," "end_block INTEGER," "root BLOB," "PRIMARY KEY(level, idx)" ");", p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName ); /* Unless a malloc() failure has occurred, execute the SQL script to ** create the tables used to store data for this FTS3 virtual table. */ if( zContentCols==0 || zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); } sqlite3_free(zSql); sqlite3_free(zContentCols); return rc; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** ** argv[0] -> module name ** argv[1] -> database name ** argv[2] -> table name ** argv[...] -> "column name" fields... */ int fts3InitVtab( int isCreate, /* True for xCreate, false for xConnect */ sqlite3 *db, /* The SQLite database connection */ void *pAux, /* Hash table containing tokenizers */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ Fts3Hash *pHash = (Fts3Hash *)pAux; Fts3Table *p; /* Pointer to allocated vtab */ int rc; /* Return code */ int i; int nByte; int iCol; int nString = 0; int nCol = 0; char *zCsr; int nDb; int nName; const char *zTokenizer = 0; sqlite3_tokenizer *pTokenizer; /* Tokenizer for this table */ nDb = strlen(argv[1]) + 1; nName = strlen(argv[2]) + 1; for(i=3; i<argc; i++){ char const *z = argv[i]; rc = sqlite3Fts3InitTokenizer(pHash, z, &pTokenizer, &zTokenizer, pzErr); if( rc!=SQLITE_OK ){ return rc; } if( z!=zTokenizer ){ nString += strlen(z) + 1; } } nCol = argc - 3 - (zTokenizer!=0); if( zTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr); if( rc!=SQLITE_OK ){ return rc; } assert( pTokenizer ); } /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ p = (Fts3Table*)sqlite3_malloc(nByte); if( p==0 ){ rc = SQLITE_NOMEM; goto fts3_init_out; } memset(p, 0, nByte); p->db = db; p->nColumn = nCol; p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; zCsr = (char *)&p->azColumn[nCol]; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; p->zDb = zCsr; memcpy(zCsr, argv[1], nDb); zCsr += nDb; /* Fill in the azColumn array */ iCol = 0; for(i=3; i<argc; i++){ if( argv[i]!=zTokenizer ){ char *z; int n; z = (char *)sqlite3Fts3NextToken(argv[i], &n); memcpy(zCsr, z, n); zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol++] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } } /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ rc = fts3CreateTables(p); if( rc!=SQLITE_OK ) goto fts3_init_out; } rc = fts3DeclareVtab(p); if( rc!=SQLITE_OK ) goto fts3_init_out; *ppVTab = &p->base; fts3_init_out: if( rc!=SQLITE_OK ){ if( p ) fts3DisconnectMethod((sqlite3_vtab *)p); else if( pTokenizer ) pTokenizer->pModule->xDestroy(pTokenizer); } return rc; } /* ** The xConnect() and xCreate() methods for the virtual table. All the ** work is done in function fts3InitVtab(). */ static int fts3ConnectMethod( sqlite3 *db, /* Database connection */ void *pAux, /* Pointer to tokenizer hash table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); } static int fts3CreateMethod( sqlite3 *db, /* Database connection */ void *pAux, /* Pointer to tokenizer hash table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); } /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: ** ** 1. Direct lookup by rowid or docid. ** 2. Full-text search using a MATCH operator on a non-docid column. ** 3. Linear scan of %_content table. */ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts3Table *p = (Fts3Table *)pVTab; int i; /* Iterator variable */ int iCons = -1; /* Index of constraint to use */ /* By default use a full table scan. This is an expensive option, ** so search through the constraints to see if a more efficient ** strategy is possible. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; pInfo->estimatedCost = 500000; for(i=0; i<pInfo->nConstraint; i++){ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; if( pCons->usable==0 ) continue; /* A direct lookup on the rowid or docid column. This is the best ** strategy in all cases. Assign a cost of 1.0 and return early. */ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 ) ){ pInfo->idxNum = FTS3_DOCID_SEARCH; pInfo->estimatedCost = 1.0; iCons = i; break; } /* A MATCH constraint. Use a full-text search. ** ** If there is more than one MATCH constraint available, use the first ** one encountered. If there is both a MATCH constraint and a direct ** rowid/docid lookup, prefer the rowid/docid strategy. */ if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn ){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; pInfo->estimatedCost = 2.0; iCons = i; } } if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = 1; pInfo->aConstraintUsage[iCons].omit = 1; } return SQLITE_OK; } /* ** Implementation of xOpen method. */ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ /* Allocate a buffer large enough for an Fts3Cursor structure. If the ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, ** if the allocation fails, return SQLITE_NOMEM. */ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); if( !pCsr ){ return SQLITE_NOMEM; } memset(pCsr, 0, sizeof(Fts3Cursor)); return SQLITE_OK; } /****************************************************************/ /****************************************************************/ /****************************************************************/ /****************************************************************/ /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fulltextClose(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr); return SQLITE_OK; } static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ int rc; /* Return code */ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; if( pCsr->aDoclist==0 ){ if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ rc = SQLITE_OK; }else{ pCsr->isEof = 1; rc = sqlite3_reset(pCsr->pStmt); } }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ pCsr->isEof = 1; rc = SQLITE_OK; }else{ sqlite3_reset(pCsr->pStmt); fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ rc = SQLITE_OK; }else{ pCsr->isEof = 1; if( SQLITE_OK==(rc = sqlite3_reset(pCsr->pStmt)) ){ rc = SQLITE_ERROR; } } } return rc; } /* ** The buffer pointed to by argument zNode (size nNode bytes) contains the ** root node of a b-tree segment. The segment is guaranteed to be at least ** one level high (i.e. the root node is not also a leaf). If successful, ** this function locates the leaf node of the segment that may contain the ** term specified by arguments zTerm and nTerm and writes its block number ** to *piLeaf. ** ** It is possible that the returned leaf node does not contain the specified ** term. However, if the segment does contain said term, it is stored on ** the identified leaf node. Because this function only inspects interior ** segment nodes (and never loads leaf nodes into memory), it is not possible ** to be sure. ** ** If an error occurs, an error code other than SQLITE_OK is returned. */ static int fts3SelectLeaf( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ sqlite3_int64 *piLeaf /* Selected leaf node */ ){ int rc = SQLITE_OK; /* Return code */ const char *zCsr = zNode; /* Cursor to iterate through node */ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ char *zBuffer = 0; /* Buffer to load terms into */ int nAlloc = 0; /* Size of allocated buffer */ while( 1 ){ int iHeight; /* Height of this node in tree */ sqlite3_int64 iChild; /* Block id of child node to descend to */ int nBlock; /* Size of child node in bytes */ zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); while( zCsr<zEnd ){ int cmp; /* memcmp() result */ int nSuffix; /* Size of term suffix */ int nPrefix = 0; /* Size of term prefix */ int nBuffer; /* Total term size */ /* Load the next term on the node into zBuffer */ if( zBuffer ){ zCsr += sqlite3Fts3GetVarint32(zCsr, &nPrefix); } zCsr += sqlite3Fts3GetVarint32(zCsr, &nSuffix); if( nPrefix+nSuffix>nAlloc ){ char *zNew; nAlloc = (nPrefix+nSuffix) * 2; zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); if( !zNew ){ sqlite3_free(zBuffer); return SQLITE_NOMEM; } zBuffer = zNew; } memcpy(&zBuffer[nPrefix], zCsr, nSuffix); nBuffer = nPrefix + nSuffix; zCsr += nSuffix; /* Compare the term we are searching for with the term just loaded from ** the interior node. If the specified term is greater than or equal ** to the term from the interior node, then all terms on the sub-tree ** headed by node iChild are smaller than zTerm. No need to search ** iChild. ** ** If the interior node term is larger than the specified term, then ** the tree headed by iChild may contain the specified term. */ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; iChild++; }; /* If (iHeight==1), the children of this interior node are leaves. The ** specified term may be present on leaf node iChild. */ if( iHeight==1 ){ *piLeaf = iChild; break; } /* Descend to interior node iChild. */ rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); if( rc!=SQLITE_OK ) break; zEnd = &zCsr[nBlock]; } sqlite3_free(zBuffer); return rc; } static void fts3PutDeltaVarint( char **pp, sqlite3_int64 *piPrev, sqlite3_int64 iVal ){ assert( iVal-*piPrev > 0 ); *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); *piPrev = iVal; } static void fts3PoslistCopy(char **pp, char **ppPoslist){ char *pEnd = *ppPoslist; char c = 0; while( *pEnd | c ) c = *pEnd++ & 0x80; pEnd++; if( pp ){ int n = pEnd - *ppPoslist; char *p = *pp; memcpy(p, *ppPoslist, n); p += n; *pp = p; } *ppPoslist = pEnd; } static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ char *pEnd = *ppPoslist; char c = 0; while( 0xFE & (*pEnd | c) ) c = *pEnd++ & 0x80; if( pp ){ int n = pEnd - *ppPoslist; char *p = *pp; memcpy(p, *ppPoslist, n); p += n; *pp = p; } *ppPoslist = pEnd; } /* ** */ static void fts3PoslistMerge( char **pp, /* Output buffer */ char **pp1, /* Left input list */ char **pp2 /* Right input list */ ){ char *p = *pp; char *p1 = *pp1; char *p2 = *pp2; while( *p1 && *p2 ){ int iCol1 = 0; int iCol2 = 0; if( *p1==0x01 ) sqlite3Fts3GetVarint32(&p1[1], &iCol1); if( *p2==0x01 ) sqlite3Fts3GetVarint32(&p2[1], &iCol2); if( iCol1==iCol2 ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; if( iCol1!=0 ){ int n; *p++ = 0x01; n = sqlite3Fts3PutVarint(p, iCol1); p += n; p1 += 1 + n; p2 += 1 + n; } while( (*p1&0xFE) || (*p2&0xFE) ){ if( i1==i2 ){ fts3GetDeltaVarint(&p1, &i1); i1 -= 2; fts3GetDeltaVarint(&p2, &i2); i2 -= 2; }else if( i1<i2 ){ fts3GetDeltaVarint(&p1, &i1); i1 -= 2; }else{ fts3GetDeltaVarint(&p2, &i2); i2 -= 2; } fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2; if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF; if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF; } }else if( iCol1<iCol2 ){ fts3ColumnlistCopy(&p, &p1); }else{ fts3ColumnlistCopy(&p, &p2); } } *p++ = '\0'; *pp = p; *pp1 = p1 + 1; *pp2 = p2 + 1; } /* ** nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* Output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ char **pp1, /* Left input list */ char **pp2 /* Right input list */ ){ char *p = (pp ? *pp : 0); char *p1 = *pp1; char *p2 = *pp2; int iCol1 = 0; int iCol2 = 0; assert( *p1!=0 && *p2!=0 ); if( *p1==0x01 ){ p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); } if( *p2==0x01 ){ p2++; p2 += sqlite3Fts3GetVarint32(p2, &iCol2); } while( 1 ){ if( iCol1==iCol2 ){ char *pSave = p; sqlite3_int64 iPrev = 0; sqlite3_int64 iPos1 = 0; sqlite3_int64 iPos2 = 0; if( pp && iCol1 ){ *p++ = 0x01; p += sqlite3Fts3PutVarint(p, iCol1); } assert( *p1!=0x00 && *p2!=0x00 && *p1!=0x01 && *p2!=0x01 ); fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; while( 1 ){ if( iPos2>iPos1 && iPos2<=iPos1+nToken ){ sqlite3_int64 iSave; if( !pp ){ fts3PoslistCopy(0, &p2); fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; return 1; } iSave = isSaveLeft ? iPos1 : iPos2; fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; pSave = 0; } if( iPos2<=iPos1 ){ if( (*p2&0xFE)==0 ) break; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; }else{ if( (*p1&0xFE)==0 ) break; fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; } } if( pSave && pp ){ p = pSave; } fts3ColumnlistCopy(0, &p1); fts3ColumnlistCopy(0, &p2); assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); if( 0==*p1 || 0==*p2 ) break; p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); p2++; p2 += sqlite3Fts3GetVarint32(p2, &iCol2); } /* Advance pointer p1 or p2 (whichever corresponds to the smaller of ** iCol1 and iCol2) so that it points to either the 0x00 that marks the ** end of the position list, or the 0x01 that precedes the next ** column-number in the position list. */ else if( iCol1<iCol2 ){ fts3ColumnlistCopy(0, &p1); if( 0==*p1 ) break; p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); }else{ fts3ColumnlistCopy(0, &p2); if( 0==*p2 ) break; p2++; p2 += sqlite3Fts3GetVarint32(p2, &iCol2); } } fts3PoslistCopy(0, &p2); fts3PoslistCopy(0, &p1); *pp1 = p1; *pp2 = p2; if( !pp || *pp==p ){ return 0; } *p++ = 0x00; *pp = p; return 1; } static int fts3PoslistNearMerge( char **pp, /* Output buffer */ char *aTmp, /* Temporary buffer space */ int nRight, /* Maximum difference in token positions */ int nLeft, /* Maximum difference in token positions */ char **pp1, /* Left input list */ char **pp2 /* Right input list */ ){ char *p1 = *pp1; char *p2 = *pp2; if( !pp ){ if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1; *pp1 = p1; *pp2 = p2; return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1); }else{ char *pTmp1 = aTmp; char *pTmp2; char *aTmp2; int res = 1; fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2); aTmp2 = pTmp2 = pTmp1; *pp1 = p1; *pp2 = p2; fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1); if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ fts3PoslistMerge(pp, &aTmp, &aTmp2); }else if( pTmp1!=aTmp ){ fts3PoslistCopy(pp, &aTmp); }else if( pTmp2!=aTmp2 ){ fts3PoslistCopy(pp, &aTmp2); }else{ res = 0; } return res; } } /* ** Values that may be used as the first parameter to fts3DoclistMerge(). */ #define MERGE_NOT 2 /* D + D -> D */ #define MERGE_AND 3 /* D + D -> D */ #define MERGE_OR 4 /* D + D -> D */ #define MERGE_POS_OR 5 /* P + P -> P */ #define MERGE_PHRASE 6 /* P + P -> D */ #define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_NEAR 8 /* P + P -> D */ #define MERGE_POS_NEAR 9 /* P + P -> P */ static int fts3DoclistMerge( int mergetype, /* One of the MERGE_XXX constants */ int nParam1, int nParam2, char *aBuffer, /* Pre-allocated output buffer */ int *pnBuffer, /* OUT: Bytes written to aBuffer */ char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ int n2 /* Size of buffer a2 */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; char *p = aBuffer; char *p1 = a1; char *p2 = a2; char *pEnd1 = &a1[n1]; char *pEnd2 = &a2[n2]; assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR ); if( !aBuffer ){ return SQLITE_NOMEM; } /* Read the first docid from each doclist */ fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); switch( mergetype ){ case MERGE_OR: case MERGE_POS_OR: while( p1 || p2 ){ if( p2 && p1 && i1==i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( !p2 || (p1 && i1<i2) ){ fts3PutDeltaVarint(&p, &iPrev, i1); if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3PutDeltaVarint(&p, &iPrev, i2); if( mergetype==MERGE_POS_OR ) fts3PoslistCopy(&p, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; case MERGE_AND: while( p1 && p2 ){ if( i1==i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( i1<i2 ){ fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; case MERGE_NOT: while( p1 ){ if( p2 && i1==i2 ){ fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( !p2 || i1<i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; case MERGE_POS_PHRASE: case MERGE_PHRASE: { char **ppPos = (mergetype==MERGE_PHRASE ? 0 : &p); while( p1 && p2 ){ if( i1==i2 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; fts3PutDeltaVarint(&p, &iPrev, i1); if( 0==fts3PoslistPhraseMerge(ppPos, 1, 0, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; } fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( i1<i2 ){ fts3PoslistCopy(0, &p1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; } case MERGE_POS_NEAR: case MERGE_NEAR: { char *aTmp = 0; char **ppPos = 0; if( mergetype==MERGE_POS_NEAR ){ ppPos = &p; aTmp = sqlite3_malloc(2*(n1+n2)); if( !aTmp ){ return SQLITE_NOMEM; } } while( p1 && p2 ){ if( i1==i2 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; fts3PutDeltaVarint(&p, &iPrev, i1); if( !fts3PoslistNearMerge(ppPos, aTmp, nParam1, nParam2, &p1, &p2) ){ iPrev = iPrevSave; p = pSave; } fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); }else if( i1<i2 ){ fts3PoslistCopy(0, &p1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); }else{ fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } sqlite3_free(aTmp); break; } default: assert(!"Invalid mergetype value passed to fts3DoclistMerge()"); } *pnBuffer = (p-aBuffer); return SQLITE_OK; } /* ** A pointer to an instance of this structure is used as the context ** argument to sqlite3Fts3SegReaderIterate() */ typedef struct TermSelect TermSelect; struct TermSelect { int isReqPos; char *aOutput; /* Malloc'd output buffer */ int nOutput; /* Size of output in bytes */ }; static int fts3TermSelectCb( Fts3Table *p, /* Virtual table object */ void *pContext, /* Pointer to TermSelect structure */ char *zTerm, int nTerm, char *aDoclist, int nDoclist ){ TermSelect *pTS = (TermSelect *)pContext; int nNew = pTS->nOutput + nDoclist; char *aNew = sqlite3_malloc(nNew); if( !aNew ){ return SQLITE_NOMEM; } if( pTS->nOutput==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). TODO: Add a way to transfer control of the ** aDoclist buffer from the caller so as to avoid the memcpy(). */ memcpy(aNew, aDoclist, nDoclist); }else{ /* The output buffer is not empty. Merge doclist aDoclist with the ** existing output. This can only happen with prefix-searches (as ** searches for exact terms return exactly one doclist). */ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist ); } sqlite3_free(pTS->aOutput); pTS->aOutput = aNew; pTS->nOutput = nNew; return SQLITE_OK; } /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** ** The returned doclist may be in one of two formats, depending on the ** value of parameter isReqPos. If isReqPos is zero, then the doclist is ** a sorted list of delta-compressed docids. If isReqPos is non-zero, ** then the returned list is in the same format as is stored in the ** database without the found length specifier at the start of on-disk ** doclists. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ int iColumn, /* Column to query (or -ve for all columns) */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int i; TermSelect tsc; Fts3SegFilter filter; /* Segment term filter configuration */ Fts3SegReader **apSegment = 0; /* Array of segments to read data from */ int nSegment = 0; /* Size of apSegment array */ int nAlloc = 0; /* Allocated size of segment array */ int rc; /* Return code */ sqlite3_stmt *pStmt; /* SQL statement to scan %_segdir table */ int iAge = 0; /* Used to assign ages to segments */ /* Loop through the entire %_segdir table. For each segment, create a ** Fts3SegReader to iterate through the subset of the segment leaves ** that may contain a term that matches zTerm/nTerm. For non-prefix ** searches, this is always a single leaf. For prefix searches, this ** may be a contiguous block of leaves. ** ** The code in this loop does not actually load any leaves into memory ** (unless the root node happens to be a leaf). It simply examines the ** b-tree structure to determine which leaves need to be inspected. */ rc = sqlite3Fts3AllSegdirs(p, &pStmt); while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pNew = 0; int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); if( sqlite3_column_int64(pStmt, 1)==0 ){ /* The entire segment is stored on the root node (which must be a ** leaf). Do not bother inspecting any data in this case, just ** create a Fts3SegReader to scan the single leaf. */ rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); }else{ sqlite3_int64 i1; rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); if( rc==SQLITE_OK ){ sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); } } iAge++; /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ assert( (rc==SQLITE_OK)==(pNew!=0) ); if( pNew ){ if( nSegment==nAlloc ){ nAlloc += 16; Fts3SegReader **pArray = (Fts3SegReader **)sqlite3_realloc( apSegment, nAlloc*sizeof(Fts3SegReader *) ); if( !pArray ){ sqlite3Fts3SegReaderFree(pNew); rc = SQLITE_NOMEM; goto finished; } apSegment = pArray; } apSegment[nSegment++] = pNew; } } if( rc!=SQLITE_DONE ){ assert( rc!=SQLITE_OK ); goto finished; } memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = zTerm; filter.nTerm = nTerm; rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, fts3TermSelectCb, (void *)&tsc ); if( rc==SQLITE_OK ){ *ppOut = tsc.aOutput; *pnOut = tsc.nOutput; }else{ sqlite3_free(tsc.aOutput); } finished: sqlite3_reset(pStmt); for(i=0; i<nSegment; i++){ sqlite3Fts3SegReaderFree(apSegment[i]); } sqlite3_free(apSegment); return rc; } /* ** Return a DocList corresponding to the phrase *pPhrase. ** ** The resulting DL_DOCIDS doclist is stored in pResult, which is ** overwritten. */ static int fts3PhraseSelect( Fts3Table *p, /* Virtual table handle */ Fts3Phrase *pPhrase, /* Phrase to return a doclist for */ int isReqPos, /* True if output should contain positions */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut /* OUT: Size of buffer at *paOut */ ){ char *pOut = 0; int nOut = 0; int rc = SQLITE_OK; int ii; int iCol = pPhrase->iColumn; int isTermPos = (pPhrase->nToken>1 || isReqPos); assert( p->nPendingData==0 ); for(ii=0; ii<pPhrase->nToken; ii++){ struct PhraseToken *pTok = &pPhrase->aToken[ii]; char *z = pTok->z; /* Next token of the phrase */ int n = pTok->n; /* Size of z in bytes */ int isPrefix = pTok->isPrefix;/* True if token is a prefix */ char *pList; /* Pointer to token doclist */ int nList; /* Size of buffer at pList */ rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); if( rc!=SQLITE_OK ) break; if( ii==0 ){ pOut = pList; nOut = nList; }else{ /* Merge the new term list and the current output. If this is the ** last term in the phrase, and positions are not required in the ** output of this function, the positions can be dropped as part ** of this merge. Either way, the result of this merge will be ** smaller than nList bytes. The code in fts3DoclistMerge() is written ** so that it is safe to use pList as the output as well as an input ** in this case. */ int mergetype = MERGE_POS_PHRASE; if( ii==pPhrase->nToken-1 && !isReqPos ){ mergetype = MERGE_PHRASE; } fts3DoclistMerge(mergetype, 0, 0, pList, &nOut, pOut, nOut, pList, nList); sqlite3_free(pOut); pOut = pList; } } if( rc==SQLITE_OK ){ *paOut = pOut; *pnOut = nOut; }else{ sqlite3_free(pOut); } return rc; } /* ** Evaluate the full-text expression pExpr against fts3 table pTab. Write ** the results into pRes. */ static int evalFts3Expr( Fts3Table *p, /* Virtual table handle */ Fts3Expr *pExpr, /* Parsed fts3 expression */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut /* OUT: Size of buffer at *paOut */ ){ int rc = SQLITE_OK; /* Return code */ /* Zero the output parameters. */ *paOut = 0; *pnOut = 0; if( pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ int isReqPos = (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR); rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos, paOut, pnOut); }else{ char *aLeft; char *aRight; int nLeft; int nRight; if( SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight)) && SQLITE_OK==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft)) ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: { Fts3Expr *pLeft; Fts3Expr *pRight; int mergetype = MERGE_NEAR; int nParam1; int nParam2; char *aBuffer; if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ mergetype = MERGE_POS_NEAR; } pLeft = pExpr->pLeft; while( pLeft->eType==FTSQUERY_NEAR ){ pLeft=pLeft->pRight; } pRight = pExpr->pRight; assert( pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->eType==FTSQUERY_PHRASE ); nParam1 = pExpr->nNear+1; nParam2 = nParam1+pLeft->pPhrase->nToken+pRight->pPhrase->nToken-2; aBuffer = sqlite3_malloc(nLeft+nRight+1); rc = fts3DoclistMerge(mergetype, nParam1, nParam2, aBuffer, pnOut, aLeft, nLeft, aRight, nRight ); if( rc!=SQLITE_OK ){ sqlite3_free(aBuffer); }else{ *paOut = aBuffer; } sqlite3_free(aLeft); break; } case FTSQUERY_OR: { /* Allocate a buffer for the output. The maximum size is the ** sum of the sizes of the two input buffers. The +1 term is ** so that a buffer of zero bytes is never allocated - this can ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, aLeft, nLeft, aRight, nRight ); *paOut = aBuffer; sqlite3_free(aLeft); break; } case FTSQUERY_AND: case FTSQUERY_NOT: { assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, aLeft, nLeft, aRight, nRight ); *paOut = aLeft; break; } } } sqlite3_free(aRight); } } return rc; } /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. ** ** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against ** the %_content table. ** ** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry ** in the %_content table. ** ** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The ** column on the left-hand side of the MATCH operator is column ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ /* TODO(shess) Upgrade the cursor initialization and destruction to ** account for fts3FilterMethod() being called multiple times on the ** same cursor. The current solution is very fragile. Apply fix to ** fts3 as appropriate. */ static int fts3FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */ "SELECT * FROM %Q.'%q_content'", /* full-table-scan */ }; int rc; /* Return code */ char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); } if( rc!=SQLITE_OK ) return rc; pCsr->eType = idxNum; if( idxNum==FTS3_DOCID_SEARCH ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; rc = sqlite3Fts3ExprParse(p->pTokenizer, p->azColumn, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr ); if( rc!=SQLITE_OK ) return rc; rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist); pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } if( rc!=SQLITE_OK ) return rc; return fts3NextMethod(pCursor); } /* ** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ return ((Fts3Cursor *)pCursor)->isEof; } /* This is the xColumn method of the virtual table. The SQLite ** core calls this method during a query when it needs the value ** of a column from the virtual table. This method needs to use ** one of the sqlite3_result_*() routines to store the requested ** value back in the pContext. */ static int fulltextColumn(sqlite3_vtab_cursor *pCursor, sqlite3_context *pContext, int idxCol){ Fts3Cursor *c = (Fts3Cursor *) pCursor; Fts3Table *v = cursor_vtab(c); if( idxCol<v->nColumn ){ sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1); sqlite3_result_value(pContext, pVal); }else if( idxCol==v->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor */ sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT); }else if( idxCol==v->nColumn+1 ){ /* The docid column, which is an alias for rowid. */ sqlite3_value *pVal = sqlite3_column_value(c->pStmt, 0); sqlite3_result_value(pContext, pVal); } return SQLITE_OK; } /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts3 ** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); return SQLITE_OK; } /* ** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be ** inserted, updated or deleted. */ static int fts3UpdateMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); } /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); } /* ** Implementation of xBegin() method. This is a no-op. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ assert( ((Fts3Table *)pVtab)->nPendingData==0 ); return SQLITE_OK; } /* ** Implementation of xCommit() method. This is a no-op. The contents of ** the pending-terms hash-table have already been flushed into the database ** by fts3SyncMethod(). */ static int fts3CommitMethod(sqlite3_vtab *pVtab){ assert( ((Fts3Table *)pVtab)->nPendingData==0 ); return SQLITE_OK; } /* ** Implementation of xRollback(). Discard the contents of the pending-terms ** hash-table. Any changes made to the database are reverted by SQLite. */ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ sqlite3Fts3PendingTermsClear((Fts3Table *)pVtab); return SQLITE_OK; } /* ** Helper function used by the implementation of the overloaded snippet(), ** offsets() and optimize() SQL functions. ** ** If the value passed as the third argument is a blob of size ** sizeof(Fts3Cursor*), then the blob contents are copied to the ** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error ** message is written to context pContext and SQLITE_ERROR returned. The ** string passed via zFunc is used as part of the error message. */ static int fts3FunctionArg( sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB && sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) ){ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); sqlite3_result_error(pContext, zErr, -1); sqlite3_free(zErr); return SQLITE_ERROR; } memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); *ppCsr = pRet; return SQLITE_OK; } /* ** Implementation of the snippet() function for FTS3 */ static void fts3SnippetFunc( sqlite3_context *pContext, int argc, sqlite3_value **argv ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ const char *zStart = "<b>"; const char *zEnd = "</b>"; const char *zEllipsis = "<b>...</b>"; if( argc<1 || argc>4 ) return; if( fts3FunctionArg(pContext, "snippet", argv[0], &pCsr) ) return; switch( argc ){ case 4: zEllipsis = (const char*)sqlite3_value_text(argv[3]); case 3: zEnd = (const char*)sqlite3_value_text(argv[2]); case 2: zStart = (const char*)sqlite3_value_text(argv[1]); } sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis); } /* ** Implementation of the offsets() function for FTS3 */ static void fts3OffsetsFunc( sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of argument array */ sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ assert( nVal==1 ); if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; assert( pCsr ); sqlite3Fts3Offsets(pContext, pCsr); } /* ** Implementation of the special optimize() function for FTS3. This ** function merges all segments in the database to a single segment. ** Example usage is: ** ** SELECT optimize(t) FROM t LIMIT 1; ** ** where 't' is the name of an FTS3 table. */ static void fts3OptimizeFunc( sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of argument array */ sqlite3_value **apVal /* Array of arguments */ ){ int rc; /* Return code */ Fts3Table *p; /* Virtual table handle */ Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ assert( nVal==1 ); if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; p = (Fts3Table *)pCursor->base.pVtab; assert( p ); rc = sqlite3Fts3Optimize(p); switch( rc ){ case SQLITE_OK: sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); break; case SQLITE_DONE: sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); break; default: sqlite3_result_error_code(pContext, rc); break; } } /* ** This routine implements the xFindFunction method for the FTS3 ** virtual table. */ static int fts3FindFunctionMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ int nArg, /* Number of SQL function arguments */ const char *zName, /* Name of SQL function */ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* Unused */ ){ struct Overloaded { const char *zName; void (*xFunc)(sqlite3_context*,int,sqlite3_value**); } aOverload[] = { { "snippet", fts3SnippetFunc }, { "offsets", fts3OffsetsFunc }, { "optimize", fts3OptimizeFunc }, }; int i; /* Iterator variable */ for(i=0; i<SizeofArray(aOverload); i++){ if( strcmp(zName, aOverload[i].zName)==0 ){ *pxFunc = aOverload[i].xFunc; return 1; } } /* No function of the specified name was found. Return 0. */ return 0; } /* ** Implementation of FTS3 xRename method. Rename an fts3 table. */ static int fts3RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_NOMEM; /* Return Code */ char *zSql; /* SQL script to run to rename tables */ zSql = sqlite3_mprintf( "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';" "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';" "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';" , p->zDb, p->zName, zName , p->zDb, p->zName, zName , p->zDb, p->zName, zName ); if( zSql ){ rc = sqlite3_exec(p->db, zSql, 0, 0, 0); sqlite3_free(zSql); } return rc; } static const sqlite3_module fts3Module = { /* iVersion */ 0, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, /* xClose */ fulltextClose, /* xFilter */ fts3FilterMethod, /* xNext */ fts3NextMethod, /* xEof */ fts3EofMethod, /* xColumn */ fulltextColumn, /* xRowid */ fts3RowidMethod, /* xUpdate */ fts3UpdateMethod, /* xBegin */ fts3BeginMethod, /* xSync */ fts3SyncMethod, /* xCommit */ fts3CommitMethod, /* xRollback */ fts3RollbackMethod, /* xFindFunction */ fts3FindFunctionMethod, /* xRename */ fts3RenameMethod, }; /* ** This function is registered as the module destructor (called when an ** FTS3 enabled database connection is closed). It frees the memory ** allocated for the tokenizer hash table. */ static void hashDestroy(void *p){ Fts3Hash *pHash = (Fts3Hash *)p; sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* ** The fts3 built-in tokenizers - "simple" and "porter" - are implemented ** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following ** two forward declarations are for functions declared in these files ** used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed ** to by the argument to point a the "simple" tokenizer implementation. ** Function ...PorterTokenizerModule() sets *pModule to point to the ** porter tokenizer/stemmer implementation. */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); /* ** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. */ int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); #ifdef SQLITE_ENABLE_ICU sqlite3Fts3IcuTokenizerModule(&pIcu); #endif /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); } /* Load the built-in tokenizers into the hash table */ |
︙ | ︙ | |||
6985 6986 6987 6988 6989 6990 6991 | /* Create the virtual table wrapper around the hash-table and overload ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) | | | < < < < | 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 | /* Create the virtual table wrapper around the hash-table and overload ** the two scalar functions. If this is successful, register the ** module with sqlite. */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ return sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); } /* An error has occurred. Delete the hash table and return the error code. */ |
︙ | ︙ | |||
7017 7018 7019 7020 7021 7022 7023 | const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi) return sqlite3Fts3Init(db); } #endif | | | 2196 2197 2198 2199 2200 2201 2202 2203 | const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi) return sqlite3Fts3Init(db); } #endif #endif |
Added ext/fts3/fts3Int.h.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | /* ** 2009 Nov 12 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** */ #ifndef _FTSINT_H #define _FTSINT_H #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif #include <sqlite3.h> #include "fts3_tokenizer.h" #include "fts3_hash.h" /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single ** segment of level N+1. */ #define FTS3_MERGE_COUNT 16 /* ** This is the maximum amount of data (in bytes) to store in the ** Fts3Table.pendingTerms hash table. Normally, the hash table is ** populated as documents are inserted/updated/deleted in a transaction ** and used to create a new segment when the transaction is committed. ** However if this limit is reached midway through a transaction, a new ** segment is created and the hash table cleared immediately. */ #define FTS3_MAX_PENDING_DATA (1*1024*1024) /* ** Macro to return the number of elements in an array. SQLite has a ** similar macro called ArraySize(). Use a different name to avoid ** a collision when building an amalgamation with built-in FTS3. */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3SegReader Fts3SegReader; typedef struct Fts3SegFilter Fts3SegFilter; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. ** All other methods receive a pointer to the structure as one of their ** arguments. */ struct Fts3Table { sqlite3_vtab base; /* Base class used by SQLite core */ sqlite3 *db; /* The database connection */ const char *zDb; /* logical database name */ const char *zName; /* virtual table name */ int nColumn; /* number of columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[18]; /* Pointer to string containing the SQL: ** ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? ** ORDER BY blockid" */ char *zSelectLeaves; /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds FTS3_MAX_PENDING_DATA, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. */ int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; }; /* ** When the core wants to read from the virtual table, it creates a ** virtual table cursor (an instance of the following structure) using ** the xOpen method. Cursors are destroyed using the xClose method. */ struct Fts3Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ int eType; /* Search strategy (see below) */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ int isEof; /* True if at End Of Results */ Fts3Expr *pExpr; /* Parsed MATCH query string */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ }; /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in "...", nToken will be the number ** of tokens in the string. */ struct Fts3Phrase { int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ int isNot; /* Phrase prefixed by unary not (-) operator */ struct PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer pointed to by z */ int isPrefix; /* True if token ends in with a "*" character */ } aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ int nNear; /* Valid if eType==FTSQUERY_NEAR */ Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ }; /* ** Candidate values for Fts3Query.eType. Note that the order of the first ** four values is in order of precedence when parsing expressions. For ** example, the following: ** ** "a OR b AND c NOT d NEAR e" ** ** is equivalent to: ** ** "a OR (b AND (c NOT (d NEAR e)))" */ #define FTSQUERY_NEAR 1 #define FTSQUERY_NOT 2 #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 /* fts3_init.c */ int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, sqlite3_vtab **, char **); /* fts3_write.c */ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, int (*)(Fts3Table *, void *, char *, int, char *, int), void * ); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 /* Type passed as 4th argument to SegmentReaderIterate() */ struct Fts3SegFilter { const char *zTerm; int nTerm; int iCol; int flags; }; /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, sqlite3_tokenizer **, const char **, char ** ); /* fts3_snippet.c */ void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); void sqlite3Fts3Snippet(sqlite3_context*, Fts3Cursor*, const char *, const char *, const char * ); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, const char *, int, Fts3Expr ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST void sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif #endif /* _FTSINT_H */ |
Changes to ext/fts3/fts3_expr.c.
︙ | ︙ | |||
9 10 11 12 13 14 15 | ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This module contains code that implements a parser for fts3 query strings ** (the right-hand argument to the MATCH operator). Because the supported ** syntax is relatively simple, the whole tokenizer/parser system is | | < | 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 | ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This module contains code that implements a parser for fts3 query strings ** (the right-hand argument to the MATCH operator). Because the supported ** syntax is relatively simple, the whole tokenizer/parser system is ** hand-coded. */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* ** By default, this module parses the legacy syntax that has been ** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS ** is defined, then it uses the new syntax. The differences between |
︙ | ︙ | |||
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | ** AND operators have a higher precedence than OR. ** ** If compiled with SQLITE_TEST defined, then this module exports the ** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable ** to zero causes the module to use the old syntax. If it is set to ** non-zero the new syntax is activated. This is so both syntaxes can ** be tested using a single build of testfixture. */ #ifdef SQLITE_TEST int sqlite3_fts3_enable_parentheses = 0; #else # ifdef SQLITE_ENABLE_FTS3_PARENTHESIS # define sqlite3_fts3_enable_parentheses 1 # else # define sqlite3_fts3_enable_parentheses 0 # endif #endif /* ** Default span for NEAR operators. */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 | > > > > > > > > > > > > > > > > > > > > > > | < | 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | ** AND operators have a higher precedence than OR. ** ** If compiled with SQLITE_TEST defined, then this module exports the ** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable ** to zero causes the module to use the old syntax. If it is set to ** non-zero the new syntax is activated. This is so both syntaxes can ** be tested using a single build of testfixture. ** ** The following describes the syntax supported by the fts3 MATCH ** operator in a similar format to that used by the lemon parser ** generator. This module does not use actually lemon, it uses a ** custom parser. ** ** query ::= andexpr (OR andexpr)*. ** ** andexpr ::= notexpr (AND? notexpr)*. ** ** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. ** notexpr ::= LP query RP. ** ** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. ** ** distance_opt ::= . ** distance_opt ::= / INTEGER. ** ** phrase ::= TOKEN. ** phrase ::= COLUMN:TOKEN. ** phrase ::= "TOKEN TOKEN TOKEN...". */ #ifdef SQLITE_TEST int sqlite3_fts3_enable_parentheses = 0; #else # ifdef SQLITE_ENABLE_FTS3_PARENTHESIS # define sqlite3_fts3_enable_parentheses 1 # else # define sqlite3_fts3_enable_parentheses 0 # endif #endif /* ** Default span for NEAR operators. */ #define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 #include "fts3Int.h" #include <ctype.h> #include <string.h> #include <assert.h> typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ |
︙ | ︙ | |||
350 351 352 353 354 355 356 357 358 359 360 361 362 363 | ** parenthesis, a quote character, or EOF. */ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; *pnConsumed = (zInput - z) + nKey; return SQLITE_OK; } | > > > | 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | ** parenthesis, a quote character, or EOF. */ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); if( !pRet ){ return SQLITE_NOMEM; } memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; *pnConsumed = (zInput - z) + nKey; return SQLITE_OK; } |
︙ | ︙ |
Deleted ext/fts3/fts3_expr.h.
|
| < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < |
Changes to ext/fts3/fts3_hash.c.
︙ | ︙ | |||
52 53 54 55 56 57 58 | ** "pNew" is a pointer to the hash table that is to be initialized. ** keyClass is one of the constants ** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass ** determines what kind of key the hash table will use. "copyKey" is ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. */ | | | | | | 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 | ** "pNew" is a pointer to the hash table that is to be initialized. ** keyClass is one of the constants ** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass ** determines what kind of key the hash table will use. "copyKey" is ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. */ void sqlite3Fts3HashInit(Fts3Hash *pNew, int keyClass, int copyKey){ assert( pNew!=0 ); assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); pNew->keyClass = keyClass; pNew->copyKey = copyKey; pNew->first = 0; pNew->count = 0; pNew->htsize = 0; pNew->ht = 0; } /* Remove all entries from a hash table. Reclaim all memory. ** Call this routine to delete a hash table or to reset a hash table ** to the empty state. */ void sqlite3Fts3HashClear(Fts3Hash *pH){ Fts3HashElem *elem; /* For looping over all elements of the table */ assert( pH!=0 ); elem = pH->first; pH->first = 0; fts3HashFree(pH->ht); pH->ht = 0; pH->htsize = 0; while( elem ){ Fts3HashElem *next_elem = elem->next; if( pH->copyKey && elem->pKey ){ fts3HashFree(elem->pKey); } fts3HashFree(elem); elem = next_elem; } pH->count = 0; |
︙ | ︙ | |||
160 161 162 163 164 165 166 | return &fts3BinCompare; } } /* Link an element into the hash table */ static void fts3HashInsertElement( | | | | | 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | return &fts3BinCompare; } } /* Link an element into the hash table */ static void fts3HashInsertElement( Fts3Hash *pH, /* The complete hash table */ struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ Fts3HashElem *pNew /* The element to be inserted */ ){ Fts3HashElem *pHead; /* First element already in pEntry */ pHead = pEntry->chain; if( pHead ){ pNew->next = pHead; pNew->prev = pHead->prev; if( pHead->prev ){ pHead->prev->next = pNew; } else { pH->first = pNew; } pHead->prev = pNew; |
︙ | ︙ | |||
187 188 189 190 191 192 193 | } /* Resize the hash table so that it cantains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. */ | | | | | | | 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | } /* Resize the hash table so that it cantains "new_size" buckets. ** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. */ static void fts3Rehash(Fts3Hash *pH, int new_size){ struct _fts3ht *new_ht; /* The new hash table */ Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ int (*xHash)(const void*,int); /* The hash function */ assert( (new_size & (new_size-1))==0 ); new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); if( new_ht==0 ) return; fts3HashFree(pH->ht); pH->ht = new_ht; pH->htsize = new_size; xHash = ftsHashFunction(pH->keyClass); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); next_elem = elem->next; fts3HashInsertElement(pH, &new_ht[h], elem); } } /* This function (for internal use only) locates an element in an ** hash table that matches the given key. The hash for this key has ** already been computed and is passed as the 4th parameter. */ static Fts3HashElem *fts3FindElementByHash( const Fts3Hash *pH, /* The pH to be searched */ const void *pKey, /* The key we are searching for */ int nKey, int h /* The hash for this key. */ ){ Fts3HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ int (*xCompare)(const void*,int,const void*,int); /* comparison function */ if( pH->ht ){ struct _fts3ht *pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; |
︙ | ︙ | |||
239 240 241 242 243 244 245 | return 0; } /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ static void fts3RemoveElementByHash( | | | | 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | return 0; } /* Remove a single entry from the hash table given a pointer to that ** element and a hash on the element's key. */ static void fts3RemoveElementByHash( Fts3Hash *pH, /* The pH containing "elem" */ Fts3HashElem* elem, /* The element to be removed from the pH */ int h /* Hash value for the element */ ){ struct _fts3ht *pEntry; if( elem->prev ){ elem->prev->next = elem->next; }else{ pH->first = elem->next; |
︙ | ︙ | |||
276 277 278 279 280 281 282 | } } /* Attempt to locate an element of the hash table pH with a key ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ | | | | 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 | } } /* Attempt to locate an element of the hash table pH with a key ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. */ void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ int h; /* A hash on key */ Fts3HashElem *elem; /* The element that matches key */ int (*xHash)(const void*,int); /* The hash function */ if( pH==0 || pH->ht==0 ) return 0; xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); h = (*xHash)(pKey,nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); |
︙ | ︙ | |||
306 307 308 309 310 311 312 | ** The key is not copied in this instance. If a malloc fails, then ** the new data is returned and the hash table is unchanged. ** ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ void *sqlite3Fts3HashInsert( | | | | | 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 | ** The key is not copied in this instance. If a malloc fails, then ** the new data is returned and the hash table is unchanged. ** ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ void *sqlite3Fts3HashInsert( Fts3Hash *pH, /* The hash table to insert into */ const void *pKey, /* The key */ int nKey, /* Number of bytes in the key */ void *data /* The data */ ){ int hraw; /* Raw hash value of the key */ int h; /* the hash of the key modulo hash table size */ Fts3HashElem *elem; /* Used to loop thru the element list */ Fts3HashElem *new_elem; /* New element added to the pH */ int (*xHash)(const void*,int); /* The hash function */ assert( pH!=0 ); xHash = ftsHashFunction(pH->keyClass); assert( xHash!=0 ); hraw = (*xHash)(pKey, nKey); assert( (pH->htsize & (pH->htsize-1))==0 ); |
︙ | ︙ | |||
341 342 343 344 345 346 347 | if( pH->htsize==0 ){ fts3Rehash(pH,8); if( pH->htsize==0 ){ pH->count = 0; return data; } } | | | 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 | if( pH->htsize==0 ){ fts3Rehash(pH,8); if( pH->htsize==0 ){ pH->count = 0; return data; } } new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); if( new_elem==0 ) return data; if( pH->copyKey && pKey!=0 ){ new_elem->pKey = fts3HashMalloc( nKey ); if( new_elem->pKey==0 ){ fts3HashFree(new_elem); return data; } |
︙ | ︙ |
Changes to ext/fts3/fts3_hash.h.
︙ | ︙ | |||
14 15 16 17 18 19 20 | ** hash table implementation for the full-text indexing module. ** */ #ifndef _FTS3_HASH_H_ #define _FTS3_HASH_H_ /* Forward declarations of structures. */ | | | | | | | | | 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | ** hash table implementation for the full-text indexing module. ** */ #ifndef _FTS3_HASH_H_ #define _FTS3_HASH_H_ /* Forward declarations of structures. */ typedef struct Fts3Hash Fts3Hash; typedef struct Fts3HashElem Fts3HashElem; /* A complete hash table is an instance of the following structure. ** The internals of this structure are intended to be opaque -- client ** code should not attempt to access or modify the fields of this structure ** directly. Change this structure only by using the routines below. ** However, many of the "procedures" and "functions" for modifying and ** accessing this structure are really macros, so we can't really make ** this structure opaque. */ struct Fts3Hash { char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ char copyKey; /* True if copy of key made on insert */ int count; /* Number of entries in this table */ Fts3HashElem *first; /* The first element of the array */ int htsize; /* Number of buckets in the hash table */ struct _fts3ht { /* the hash table */ int count; /* Number of entries with this hash */ Fts3HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; /* Each element in the hash table is an instance of the following ** structure. All elements are stored on a single doubly-linked list. ** ** Again, this structure is intended to be opaque, but it can't really ** be opaque because it is used by macros. */ struct Fts3HashElem { Fts3HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ void *pKey; int nKey; /* Key associated with this element */ }; /* ** There are 2 different modes of operation for a hash table: ** |
︙ | ︙ | |||
67 68 69 70 71 72 73 | */ #define FTS3_HASH_STRING 1 #define FTS3_HASH_BINARY 2 /* ** Access routines. To delete, insert a NULL pointer. */ | | | | | | | | 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 | */ #define FTS3_HASH_STRING 1 #define FTS3_HASH_BINARY 2 /* ** Access routines. To delete, insert a NULL pointer. */ void sqlite3Fts3HashInit(Fts3Hash*, int keytype, int copyKey); void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); void sqlite3Fts3HashClear(Fts3Hash*); /* ** Shorthand for the functions above */ #define fts3HashInit sqlite3Fts3HashInit #define fts3HashInsert sqlite3Fts3HashInsert #define fts3HashFind sqlite3Fts3HashFind #define fts3HashClear sqlite3Fts3HashClear /* ** Macros for looping over all elements of a hash table. The idiom is ** like this: ** ** Fts3Hash h; ** Fts3HashElem *p; ** ... ** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ ** SomeStructure *pData = fts3HashData(p); ** // do something with pData ** } */ #define fts3HashFirst(H) ((H)->first) |
︙ | ︙ |
Added ext/fts3/fts3_snippet.c.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 | /* ** 2009 Oct 23 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** */ #include "fts3Int.h" #include <string.h> #include <assert.h> #include <ctype.h> typedef struct Snippet Snippet; /* ** An instance of the following structure keeps track of generated ** matching-word offset information and snippets. */ struct Snippet { int nMatch; /* Total number of matches */ int nAlloc; /* Space allocated for aMatch[] */ struct snippetMatch { /* One entry for each matching term */ char snStatus; /* Status flag for use while constructing snippets */ short int iCol; /* The column that contains the match */ short int iTerm; /* The index in Query.pTerms[] of the matching term */ int iToken; /* The index of the matching document token */ short int nByte; /* Number of bytes in the term */ int iStart; /* The offset to the first character of the term */ } *aMatch; /* Points to space obtained from malloc */ char *zOffset; /* Text rendering of aMatch[] */ int nOffset; /* strlen(zOffset) */ char *zSnippet; /* Snippet text */ int nSnippet; /* strlen(zSnippet) */ }; /* It is not safe to call isspace(), tolower(), or isalnum() on ** hi-bit-set characters. This is the same solution used in the ** tokenizer. */ /* TODO(shess) The snippet-generation code should be using the ** tokenizer-generated tokens rather than doing its own local ** tokenization. */ /* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ static int safe_isspace(char c){ return (c&0x80)==0 ? isspace(c) : 0; } static int safe_isalnum(char c){ return (c&0x80)==0 ? isalnum(c) : 0; } /*******************************************************************/ /* DataBuffer is used to collect data into a buffer in piecemeal ** fashion. It implements the usual distinction between amount of ** data currently stored (nData) and buffer capacity (nCapacity). ** ** dataBufferInit - create a buffer with given initial capacity. ** dataBufferReset - forget buffer's data, retaining capacity. ** dataBufferSwap - swap contents of two buffers. ** dataBufferExpand - expand capacity without adding data. ** dataBufferAppend - append data. ** dataBufferAppend2 - append two pieces of data at once. ** dataBufferReplace - replace buffer's data. */ typedef struct DataBuffer { char *pData; /* Pointer to malloc'ed buffer. */ int nCapacity; /* Size of pData buffer. */ int nData; /* End of data loaded into pData. */ } DataBuffer; static void dataBufferInit(DataBuffer *pBuffer, int nCapacity){ assert( nCapacity>=0 ); pBuffer->nData = 0; pBuffer->nCapacity = nCapacity; pBuffer->pData = nCapacity==0 ? NULL : sqlite3_malloc(nCapacity); } static void dataBufferReset(DataBuffer *pBuffer){ pBuffer->nData = 0; } static void dataBufferExpand(DataBuffer *pBuffer, int nAddCapacity){ assert( nAddCapacity>0 ); /* TODO(shess) Consider expanding more aggressively. Note that the ** underlying malloc implementation may take care of such things for ** us already. */ if( pBuffer->nData+nAddCapacity>pBuffer->nCapacity ){ pBuffer->nCapacity = pBuffer->nData+nAddCapacity; pBuffer->pData = sqlite3_realloc(pBuffer->pData, pBuffer->nCapacity); } } static void dataBufferAppend(DataBuffer *pBuffer, const char *pSource, int nSource){ assert( nSource>0 && pSource!=NULL ); dataBufferExpand(pBuffer, nSource); memcpy(pBuffer->pData+pBuffer->nData, pSource, nSource); pBuffer->nData += nSource; } static void dataBufferAppend2(DataBuffer *pBuffer, const char *pSource1, int nSource1, const char *pSource2, int nSource2){ assert( nSource1>0 && pSource1!=NULL ); assert( nSource2>0 && pSource2!=NULL ); dataBufferExpand(pBuffer, nSource1+nSource2); memcpy(pBuffer->pData+pBuffer->nData, pSource1, nSource1); memcpy(pBuffer->pData+pBuffer->nData+nSource1, pSource2, nSource2); pBuffer->nData += nSource1+nSource2; } static void dataBufferReplace(DataBuffer *pBuffer, const char *pSource, int nSource){ dataBufferReset(pBuffer); dataBufferAppend(pBuffer, pSource, nSource); } /* StringBuffer is a null-terminated version of DataBuffer. */ typedef struct StringBuffer { DataBuffer b; /* Includes null terminator. */ } StringBuffer; static void initStringBuffer(StringBuffer *sb){ dataBufferInit(&sb->b, 100); dataBufferReplace(&sb->b, "", 1); } static int stringBufferLength(StringBuffer *sb){ return sb->b.nData-1; } static char *stringBufferData(StringBuffer *sb){ return sb->b.pData; } static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){ assert( sb->b.nData>0 ); if( nFrom>0 ){ sb->b.nData--; dataBufferAppend2(&sb->b, zFrom, nFrom, "", 1); } } static void append(StringBuffer *sb, const char *zFrom){ nappend(sb, zFrom, strlen(zFrom)); } static int endsInWhiteSpace(StringBuffer *p){ return stringBufferLength(p)>0 && safe_isspace(stringBufferData(p)[stringBufferLength(p)-1]); } /* If the StringBuffer ends in something other than white space, add a ** single space character to the end. */ static void appendWhiteSpace(StringBuffer *p){ if( stringBufferLength(p)==0 ) return; if( !endsInWhiteSpace(p) ) append(p, " "); } /* Remove white space from the end of the StringBuffer */ static void trimWhiteSpace(StringBuffer *p){ while( endsInWhiteSpace(p) ){ p->b.pData[--p->b.nData-1] = '\0'; } } /* ** Release all memory associated with the Snippet structure passed as ** an argument. */ static void fts3SnippetFree(Snippet *p){ sqlite3_free(p->aMatch); sqlite3_free(p->zOffset); sqlite3_free(p->zSnippet); sqlite3_free(p); } /* ** Append a single entry to the p->aMatch[] log. */ static void snippetAppendMatch( Snippet *p, /* Append the entry to this snippet */ int iCol, int iTerm, /* The column and query term */ int iToken, /* Matching token in document */ int iStart, int nByte /* Offset and size of the match */ ){ int i; struct snippetMatch *pMatch; if( p->nMatch+1>=p->nAlloc ){ p->nAlloc = p->nAlloc*2 + 10; p->aMatch = sqlite3_realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); if( p->aMatch==0 ){ p->nMatch = 0; p->nAlloc = 0; return; } } i = p->nMatch++; pMatch = &p->aMatch[i]; pMatch->iCol = iCol; pMatch->iTerm = iTerm; pMatch->iToken = iToken; pMatch->iStart = iStart; pMatch->nByte = nByte; } /* ** Sizing information for the circular buffer used in snippetOffsetsOfColumn() */ #define FTS3_ROTOR_SZ (32) #define FTS3_ROTOR_MASK (FTS3_ROTOR_SZ-1) /* ** Function to iterate through the tokens of a compiled expression. ** ** Except, skip all tokens on the right-hand side of a NOT operator. ** This function is used to find tokens as part of snippet and offset ** generation and we do nt want snippets and offsets to report matches ** for tokens on the RHS of a NOT. */ static int fts3NextExprToken(Fts3Expr **ppExpr, int *piToken){ Fts3Expr *p = *ppExpr; int iToken = *piToken; if( iToken<0 ){ /* In this case the expression p is the root of an expression tree. ** Move to the first token in the expression tree. */ while( p->pLeft ){ p = p->pLeft; } iToken = 0; }else{ assert(p && p->eType==FTSQUERY_PHRASE ); if( iToken<(p->pPhrase->nToken-1) ){ iToken++; }else{ iToken = 0; while( p->pParent && p->pParent->pLeft!=p ){ assert( p->pParent->pRight==p ); p = p->pParent; } p = p->pParent; if( p ){ assert( p->pRight!=0 ); p = p->pRight; while( p->pLeft ){ p = p->pLeft; } } } } *ppExpr = p; *piToken = iToken; return p?1:0; } /* ** Return TRUE if the expression node pExpr is located beneath the ** RHS of a NOT operator. */ static int fts3ExprBeneathNot(Fts3Expr *p){ Fts3Expr *pParent; while( p ){ pParent = p->pParent; if( pParent && pParent->eType==FTSQUERY_NOT && pParent->pRight==p ){ return 1; } p = pParent; } return 0; } /* ** Add entries to pSnippet->aMatch[] for every match that occurs against ** document zDoc[0..nDoc-1] which is stored in column iColumn. */ static void snippetOffsetsOfColumn( Fts3Cursor *pCur, /* The fulltest search cursor */ Snippet *pSnippet, /* The Snippet object to be filled in */ int iColumn, /* Index of fulltext table column */ const char *zDoc, /* Text of the fulltext table column */ int nDoc /* Length of zDoc in bytes */ ){ const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ Fts3Table *pVtab; /* The full text index */ int nColumn; /* Number of columns in the index */ int i, j; /* Loop counters */ int rc; /* Return code */ unsigned int match, prevMatch; /* Phrase search bitmasks */ const char *zToken; /* Next token from the tokenizer */ int nToken; /* Size of zToken */ int iBegin, iEnd, iPos; /* Offsets of beginning and end */ /* The following variables keep a circular buffer of the last ** few tokens */ unsigned int iRotor = 0; /* Index of current token */ int iRotorBegin[FTS3_ROTOR_SZ]; /* Beginning offset of token */ int iRotorLen[FTS3_ROTOR_SZ]; /* Length of token */ pVtab = (Fts3Table *)pCur->base.pVtab; nColumn = pVtab->nColumn; pTokenizer = pVtab->pTokenizer; pTModule = pTokenizer->pModule; rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); if( rc ) return; pTCursor->pTokenizer = pTokenizer; prevMatch = 0; while( !pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos) ){ Fts3Expr *pIter = pCur->pExpr; int iIter = -1; iRotorBegin[iRotor&FTS3_ROTOR_MASK] = iBegin; iRotorLen[iRotor&FTS3_ROTOR_MASK] = iEnd-iBegin; match = 0; for(i=0; i<(FTS3_ROTOR_SZ-1) && fts3NextExprToken(&pIter, &iIter); i++){ int nPhrase; /* Number of tokens in current phrase */ struct PhraseToken *pToken; /* Current token */ int iCol; /* Column index */ if( fts3ExprBeneathNot(pIter) ) continue; nPhrase = pIter->pPhrase->nToken; pToken = &pIter->pPhrase->aToken[iIter]; iCol = pIter->pPhrase->iColumn; if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue; if( pToken->n>nToken ) continue; if( !pToken->isPrefix && pToken->n<nToken ) continue; assert( pToken->n<=nToken ); if( memcmp(pToken->z, zToken, pToken->n) ) continue; if( iIter>0 && (prevMatch & (1<<i))==0 ) continue; match |= 1<<i; if( i==(FTS3_ROTOR_SZ-2) || nPhrase==iIter+1 ){ for(j=nPhrase-1; j>=0; j--){ int k = (iRotor-j) & FTS3_ROTOR_MASK; snippetAppendMatch(pSnippet, iColumn, i-j, iPos-j, iRotorBegin[k], iRotorLen[k]); } } } prevMatch = match<<1; iRotor++; } pTModule->xClose(pTCursor); } /* ** Remove entries from the pSnippet structure to account for the NEAR ** operator. When this is called, pSnippet contains the list of token ** offsets produced by treating all NEAR operators as AND operators. ** This function removes any entries that should not be present after ** accounting for the NEAR restriction. For example, if the queried ** document is: ** ** "A B C D E A" ** ** and the query is: ** ** A NEAR/0 E ** ** then when this function is called the Snippet contains token offsets ** 0, 4 and 5. This function removes the "0" entry (because the first A ** is not near enough to an E). ** ** When this function is called, the value pointed to by parameter piLeft is ** the integer id of the left-most token in the expression tree headed by ** pExpr. This function increments *piLeft by the total number of tokens ** in the expression tree headed by pExpr. ** ** Return 1 if any trimming occurs. Return 0 if no trimming is required. */ static int trimSnippetOffsets( Fts3Expr *pExpr, /* The search expression */ Snippet *pSnippet, /* The set of snippet offsets to be trimmed */ int *piLeft /* Index of left-most token in pExpr */ ){ if( pExpr ){ if( trimSnippetOffsets(pExpr->pLeft, pSnippet, piLeft) ){ return 1; } switch( pExpr->eType ){ case FTSQUERY_PHRASE: *piLeft += pExpr->pPhrase->nToken; break; case FTSQUERY_NEAR: { /* The right-hand-side of a NEAR operator is always a phrase. The ** left-hand-side is either a phrase or an expression tree that is ** itself headed by a NEAR operator. The following initializations ** set local variable iLeft to the token number of the left-most ** token in the right-hand phrase, and iRight to the right most ** token in the same phrase. For example, if we had: ** ** <col> MATCH '"abc def" NEAR/2 "ghi jkl"' ** ** then iLeft will be set to 2 (token number of ghi) and nToken will ** be set to 4. */ Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; int iLeft = *piLeft; int nNear = pExpr->nNear; int nToken = pRight->pPhrase->nToken; int jj, ii; if( pLeft->eType==FTSQUERY_NEAR ){ pLeft = pLeft->pRight; } assert( pRight->eType==FTSQUERY_PHRASE ); assert( pLeft->eType==FTSQUERY_PHRASE ); nToken += pLeft->pPhrase->nToken; for(ii=0; ii<pSnippet->nMatch; ii++){ struct snippetMatch *p = &pSnippet->aMatch[ii]; if( p->iTerm==iLeft ){ int isOk = 0; /* Snippet ii is an occurence of query term iLeft in the document. ** It occurs at position (p->iToken) of the document. We now ** search for an instance of token (iLeft-1) somewhere in the ** range (p->iToken - nNear)...(p->iToken + nNear + nToken) within ** the set of snippetMatch structures. If one is found, proceed. ** If one cannot be found, then remove snippets ii..(ii+N-1) ** from the matching snippets, where N is the number of tokens ** in phrase pRight->pPhrase. */ for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){ struct snippetMatch *p2 = &pSnippet->aMatch[jj]; if( p2->iTerm==(iLeft-1) ){ if( p2->iToken>=(p->iToken-nNear-1) && p2->iToken<(p->iToken+nNear+nToken) ){ isOk = 1; } } } if( !isOk ){ int kk; for(kk=0; kk<pRight->pPhrase->nToken; kk++){ pSnippet->aMatch[kk+ii].iTerm = -2; } return 1; } } if( p->iTerm==(iLeft-1) ){ int isOk = 0; for(jj=0; isOk==0 && jj<pSnippet->nMatch; jj++){ struct snippetMatch *p2 = &pSnippet->aMatch[jj]; if( p2->iTerm==iLeft ){ if( p2->iToken<=(p->iToken+nNear+1) && p2->iToken>(p->iToken-nNear-nToken) ){ isOk = 1; } } } if( !isOk ){ int kk; for(kk=0; kk<pLeft->pPhrase->nToken; kk++){ pSnippet->aMatch[ii-kk].iTerm = -2; } return 1; } } } break; } } if( trimSnippetOffsets(pExpr->pRight, pSnippet, piLeft) ){ return 1; } } return 0; } /* ** Compute all offsets for the current row of the query. ** If the offsets have already been computed, this routine is a no-op. */ static int snippetAllOffsets(Fts3Cursor *pCsr, Snippet **ppSnippet){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int nColumn; int iColumn, i; int iFirst, iLast; int iTerm = 0; Snippet *pSnippet; if( pCsr->pExpr==0 ){ return SQLITE_OK; } pSnippet = (Snippet *)sqlite3_malloc(sizeof(Snippet)); *ppSnippet = pSnippet; if( !pSnippet ){ return SQLITE_NOMEM; } memset(pSnippet, 0, sizeof(Snippet)); nColumn = p->nColumn; iColumn = (pCsr->eType - 2); if( iColumn<0 || iColumn>=nColumn ){ /* Look for matches over all columns of the full-text index */ iFirst = 0; iLast = nColumn-1; }else{ /* Look for matches in the iColumn-th column of the index only */ iFirst = iColumn; iLast = iColumn; } for(i=iFirst; i<=iLast; i++){ const char *zDoc; int nDoc; zDoc = (const char*)sqlite3_column_text(pCsr->pStmt, i+1); nDoc = sqlite3_column_bytes(pCsr->pStmt, i+1); snippetOffsetsOfColumn(pCsr, pSnippet, i, zDoc, nDoc); } while( trimSnippetOffsets(pCsr->pExpr, pSnippet, &iTerm) ){ iTerm = 0; } return SQLITE_OK; } /* ** Convert the information in the aMatch[] array of the snippet ** into the string zOffset[0..nOffset-1]. This string is used as ** the return of the SQL offsets() function. */ static void snippetOffsetText(Snippet *p){ int i; int cnt = 0; StringBuffer sb; char zBuf[200]; if( p->zOffset ) return; initStringBuffer(&sb); for(i=0; i<p->nMatch; i++){ struct snippetMatch *pMatch = &p->aMatch[i]; if( pMatch->iTerm>=0 ){ /* If snippetMatch.iTerm is less than 0, then the match was ** discarded as part of processing the NEAR operator (see the ** trimSnippetOffsetsForNear() function for details). Ignore ** it in this case */ zBuf[0] = ' '; sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); append(&sb, zBuf); cnt++; } } p->zOffset = stringBufferData(&sb); p->nOffset = stringBufferLength(&sb); } /* ** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set ** of matching words some of which might be in zDoc. zDoc is column ** number iCol. ** ** iBreak is suggested spot in zDoc where we could begin or end an ** excerpt. Return a value similar to iBreak but possibly adjusted ** to be a little left or right so that the break point is better. */ static int wordBoundary( int iBreak, /* The suggested break point */ const char *zDoc, /* Document text */ int nDoc, /* Number of bytes in zDoc[] */ struct snippetMatch *aMatch, /* Matching words */ int nMatch, /* Number of entries in aMatch[] */ int iCol /* The column number for zDoc[] */ ){ int i; if( iBreak<=10 ){ return 0; } if( iBreak>=nDoc-10 ){ return nDoc; } for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){} while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; } if( i<nMatch ){ if( aMatch[i].iStart<iBreak+10 ){ return aMatch[i].iStart; } if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ return aMatch[i-1].iStart; } } for(i=1; i<=10; i++){ if( safe_isspace(zDoc[iBreak-i]) ){ return iBreak - i + 1; } if( safe_isspace(zDoc[iBreak+i]) ){ return iBreak + i + 1; } } return iBreak; } /* ** Allowed values for Snippet.aMatch[].snStatus */ #define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ #define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ /* ** Generate the text of a snippet. */ static void snippetText( Fts3Cursor *pCursor, /* The cursor we need the snippet for */ Snippet *pSnippet, const char *zStartMark, /* Markup to appear before each match */ const char *zEndMark, /* Markup to appear after each match */ const char *zEllipsis /* Ellipsis mark */ ){ int i, j; struct snippetMatch *aMatch; int nMatch; int nDesired; StringBuffer sb; int tailCol; int tailOffset; int iCol; int nDoc; const char *zDoc; int iStart, iEnd; int tailEllipsis = 0; int iMatch; sqlite3_free(pSnippet->zSnippet); pSnippet->zSnippet = 0; aMatch = pSnippet->aMatch; nMatch = pSnippet->nMatch; initStringBuffer(&sb); for(i=0; i<nMatch; i++){ aMatch[i].snStatus = SNIPPET_IGNORE; } nDesired = 0; for(i=0; i<FTS3_ROTOR_SZ; i++){ for(j=0; j<nMatch; j++){ if( aMatch[j].iTerm==i ){ aMatch[j].snStatus = SNIPPET_DESIRED; nDesired++; break; } } } iMatch = 0; tailCol = -1; tailOffset = 0; for(i=0; i<nMatch && nDesired>0; i++){ if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; nDesired--; iCol = aMatch[i].iCol; zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); iStart = aMatch[i].iStart - 40; iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); if( iStart<=10 ){ iStart = 0; } if( iCol==tailCol && iStart<=tailOffset+20 ){ iStart = tailOffset; } if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ trimWhiteSpace(&sb); appendWhiteSpace(&sb); append(&sb, zEllipsis); appendWhiteSpace(&sb); } iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); if( iEnd>=nDoc-10 ){ iEnd = nDoc; tailEllipsis = 0; }else{ tailEllipsis = 1; } while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; } while( iStart<iEnd ){ while( iMatch<nMatch && aMatch[iMatch].iStart<iStart && aMatch[iMatch].iCol<=iCol ){ iMatch++; } if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd && aMatch[iMatch].iCol==iCol ){ nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart); iStart = aMatch[iMatch].iStart; append(&sb, zStartMark); nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte); append(&sb, zEndMark); iStart += aMatch[iMatch].nByte; for(j=iMatch+1; j<nMatch; j++){ if( aMatch[j].iTerm==aMatch[iMatch].iTerm && aMatch[j].snStatus==SNIPPET_DESIRED ){ nDesired--; aMatch[j].snStatus = SNIPPET_IGNORE; } } }else{ nappend(&sb, &zDoc[iStart], iEnd - iStart); iStart = iEnd; } } tailCol = iCol; tailOffset = iEnd; } trimWhiteSpace(&sb); if( tailEllipsis ){ appendWhiteSpace(&sb); append(&sb, zEllipsis); } pSnippet->zSnippet = stringBufferData(&sb); pSnippet->nSnippet = stringBufferLength(&sb); } void sqlite3Fts3Offsets( sqlite3_context *pCtx, /* SQLite function call context */ Fts3Cursor *pCsr /* Cursor object */ ){ Snippet *p; /* Snippet structure */ int rc = snippetAllOffsets(pCsr, &p); snippetOffsetText(p); sqlite3_result_text(pCtx, p->zOffset, p->nOffset, SQLITE_TRANSIENT); fts3SnippetFree(p); } void sqlite3Fts3Snippet( sqlite3_context *pCtx, /* SQLite function call context */ Fts3Cursor *pCsr, /* Cursor object */ const char *zStart, /* Snippet start text - "<b>" */ const char *zEnd, /* Snippet end text - "</b>" */ const char *zEllipsis /* Snippet ellipsis text - "<b>...</b>" */ ){ Snippet *p; /* Snippet structure */ int rc = snippetAllOffsets(pCsr, &p); snippetText(pCsr, p, zStart, zEnd, zEllipsis); sqlite3_result_text(pCtx, p->zSnippet, p->nSnippet, SQLITE_TRANSIENT); fts3SnippetFree(p); } |
Changes to ext/fts3/fts3_tokenizer.c.
︙ | ︙ | |||
26 27 28 29 30 31 32 | #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif | | | | > > > > > | 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include "sqlite3ext.h" #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #endif #include "fts3Int.h" #include <assert.h> #include <ctype.h> #include <string.h> static int safe_isspace(char c){ return (c&0x80)==0 ? isspace(c) : 0; } /* ** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** ** SELECT <function-name>(<key-name>); ** SELECT <function-name>(<key-name>, <pointer>); |
︙ | ︙ | |||
55 56 57 58 59 60 61 | ** to string <key-name> (after the hash-table is updated, if applicable). */ static void scalarFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ | | | | 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | ** to string <key-name> (after the hash-table is updated, if applicable). */ static void scalarFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; void *pPtr = 0; const unsigned char *zName; int nName; assert( argc==1 || argc==2 ); pHash = (Fts3Hash *)sqlite3_user_data(context); zName = sqlite3_value_text(argv[0]); nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); |
︙ | ︙ | |||
92 93 94 95 96 97 98 99 100 101 102 103 104 105 | sqlite3_free(zErr); return; } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } #ifdef SQLITE_TEST #include <tcl.h> #include <string.h> /* | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | sqlite3_free(zErr); return; } } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } static int fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; return (c&0x80 || isFtsIdChar[(int)(c)]); } const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ const char *z1; const char *z2 = 0; /* Find the start of the next token. */ z1 = zStr; while( z2==0 ){ switch( *z1 ){ case '\0': return 0; /* No more tokens here */ case '\'': case '"': case '`': { z2 = &z1[1]; while( *z2 && (z2[0]!=*z1 || z2[1]==*z1) ) z2++; if( *z2 ) z2++; break; } case '[': z2 = &z1[1]; while( *z2 && z2[0]!=']' ) z2++; if( *z2 ) z2++; break; default: if( fts3IsIdChar(*z1) ){ z2 = &z1[1]; while( fts3IsIdChar(*z2) ) z2++; }else{ z1++; } } } *pn = (z2-z1); return z1; } int sqlite3Fts3InitTokenizer( Fts3Hash *pHash, /* Tokenizer hash table */ const char *zArg, /* Possible tokenizer specification */ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ char **pzErr /* OUT: Set to malloced error message */ ){ int rc; char *z = (char *)zArg; int n; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; if( !z ){ zCopy = sqlite3_mprintf("simple"); }else{ while( safe_isspace(*z) ) z++; if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ return SQLITE_OK; } zCopy = sqlite3_mprintf("%s", &z[8]); *pzTokenizer = zArg; } if( !zCopy ){ return SQLITE_NOMEM; } zEnd = &zCopy[strlen(zCopy)]; z = (char *)sqlite3Fts3NextToken(zCopy, &n); z[n] = '\0'; sqlite3Fts3Dequote(z); m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, strlen(z)+1); if( !m ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); rc = SQLITE_ERROR; }else{ char const **aArg = 0; int iArg = 0; z = &z[n+1]; while( z<zEnd && (z = (char *)sqlite3Fts3NextToken(z, &n)) ){ int nNew = sizeof(char *)*(iArg+1); char const **aNew = (const char **)sqlite3_realloc(aArg, nNew); if( !aNew ){ sqlite3_free(zCopy); sqlite3_free(aArg); return SQLITE_NOMEM; } aArg = aNew; aArg[iArg++] = z; z[n] = '\0'; sqlite3Fts3Dequote(z); z = &z[n+1]; } rc = m->xCreate(iArg, aArg, ppTok); assert( rc!=SQLITE_OK || *ppTok ); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); }else{ (*ppTok)->pModule = m; } sqlite3_free(aArg); } sqlite3_free(zCopy); return rc; } #ifdef SQLITE_TEST #include <tcl.h> #include <string.h> /* |
︙ | ︙ | |||
129 130 131 132 133 134 135 | ** */ static void testFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ | | | 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 | ** */ static void testFunc( sqlite3_context *context, int argc, sqlite3_value **argv ){ Fts3Hash *pHash; sqlite3_tokenizer_module *p; sqlite3_tokenizer *pTokenizer = 0; sqlite3_tokenizer_cursor *pCsr = 0; const char *zErr = 0; const char *zName; |
︙ | ︙ | |||
162 163 164 165 166 167 168 | nInput = sqlite3_value_bytes(argv[argc-1]); zInput = (const char *)sqlite3_value_text(argv[argc-1]); if( argc==3 ){ zArg = (const char *)sqlite3_value_text(argv[1]); } | | | 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | nInput = sqlite3_value_bytes(argv[argc-1]); zInput = (const char *)sqlite3_value_text(argv[argc-1]); if( argc==3 ){ zArg = (const char *)sqlite3_value_text(argv[1]); } pHash = (Fts3Hash *)sqlite3_user_data(context); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); return; |
︙ | ︙ | |||
331 332 333 334 335 336 337 | ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ int sqlite3Fts3InitHashTable( sqlite3 *db, | | | 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 | ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ int sqlite3Fts3InitHashTable( sqlite3 *db, Fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; void *p = (void *)pHash; const int any = SQLITE_ANY; char *zTest = 0; char *zTest2 = 0; |
︙ | ︙ |
Added ext/fts3/fts3_write.c.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 | /* ** 2009 Oct 23 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This file is part of the SQLite FTS3 extension module. Specifically, ** this file contains code to insert, update and delete rows from FTS3 ** tables. It also contains code to merge FTS3 b-tree segments. Some ** of the sub-routines used to merge segments are also used by the query ** code in fts3.c. */ #include "fts3Int.h" #include <string.h> #include <assert.h> #include <stdlib.h> #define INTERIOR_MAX 2048 /* Soft limit for segment node size */ #define LEAF_MAX 2048 /* Soft limit for segment leaf size */ typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* ** Data structure used while accumulating terms in the pending-terms hash ** table. The hash table entry maps from term (a string) to a malloced ** instance of this structure. */ struct PendingList { int nData; char *aData; int nSpace; sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; }; /* ** An instance of this structure is used to iterate through the terms on ** a contiguous set of segment b-tree leaf nodes. Although the details of ** this structure are only manipulated by code in this file, opaque handles ** of type Fts3SegReader* are also used by code in fts3.c to iterate through ** terms when querying the full-text index. See functions: ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() ** sqlite3Fts3SegReaderIterate() */ struct Fts3SegReader { int iIdx; /* Index within level */ sqlite3_int64 iStartBlock; sqlite3_int64 iEndBlock; sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ int nTermAlloc; /* Allocated size of zTerm buffer */ /* Variables set by fts3SegReaderNext(). These may be read directly ** by the caller. They are valid from the time SegmentReaderNew() returns ** until SegmentReaderNext() returns something other than SQLITE_OK ** (i.e. SQLITE_DONE). */ int nTerm; /* Number of bytes in current term */ char *zTerm; /* Pointer to current term */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ /* The following variables are used to iterate through the current doclist */ char *pOffsetList; sqlite3_int64 iDocid; }; /* ** An instance of this structure is used to create a segment b-tree in the ** database. The internal details of this type are only accessed by the ** following functions: ** ** fts3SegWriterAdd() ** fts3SegWriterFlush() ** fts3SegWriterFree() */ struct SegmentWriter { SegmentNode *pTree; /* Pointer to interior tree structure */ sqlite3_int64 iFirst; /* First slot in %_segments written */ sqlite3_int64 iFree; /* Next free slot in %_segments */ char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except ** the leaf nodes). These functions and type are only ever used by code ** within the fts3SegWriterXXX() family of functions described above. ** ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ SegmentNode *pRight; /* Pointer to right-sibling */ SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ int nEntry; /* Number of terms written to node so far */ char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nData; /* Bytes of valid data so far */ char *aData; /* Node data */ }; /* ** Valid values for the second argument to fts3SqlStmt(). */ #define SQL_DELETE_CONTENT 0 #define SQL_IS_EMPTY 1 #define SQL_DELETE_ALL_CONTENT 2 #define SQL_DELETE_ALL_SEGMENTS 3 #define SQL_DELETE_ALL_SEGDIR 4 #define SQL_SELECT_CONTENT_BY_ROWID 5 #define SQL_NEXT_SEGMENT_INDEX 6 #define SQL_INSERT_SEGMENTS 7 #define SQL_NEXT_SEGMENTS_ID 8 #define SQL_INSERT_SEGDIR 9 #define SQL_SELECT_LEVEL 10 #define SQL_SELECT_ALL_LEVEL 11 #define SQL_SELECT_LEVEL_COUNT 12 #define SQL_SELECT_SEGDIR_COUNT_MAX 13 #define SQL_DELETE_SEGDIR_BY_LEVEL 14 #define SQL_DELETE_SEGMENTS_RANGE 15 #define SQL_CONTENT_INSERT 16 #define SQL_GET_BLOCK 17 static int fts3SqlStmt( Fts3Table *p, int eStmt, sqlite3_stmt **pp, sqlite3_value **apVal ){ const char *azSql[] = { /* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", /* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", /* 6 */ "SELECT coalesce(max(idx)+1, 0) FROM %Q.'%q_segdir' WHERE level=?", /* 7 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 8 */ "SELECT coalesce(max(blockid)+1, 1) FROM %Q.'%q_segments'", /* 9 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 10 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 11 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", /* 12 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 13 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 14 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 16 */ 0, /* CONTENT_INSERT - generated elsewhere */ /* 17 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); pStmt = p->aStmt[eStmt]; if( !pStmt ){ char *zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; } } if( apVal ){ int i; int nParam = sqlite3_bind_parameter_count(pStmt); for(i=0; rc==SQLITE_OK && i<nParam; i++){ rc = sqlite3_bind_value(pStmt, i+1, apVal[i]); } } *pp = pStmt; return rc; } /* ** Read a single block from the %_segments table. If the specified block ** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO ** etc.) occurs, return the appropriate SQLite error code. ** ** Otherwise, if successful, set *pzBlock to point to a buffer containing ** the block read from the database, and *pnBlock to the size of the read ** block in bytes. ** ** WARNING: The returned buffer is only valid until the next call to ** sqlite3Fts3ReadBlock(). */ int sqlite3Fts3ReadBlock( Fts3Table *p, sqlite3_int64 iBlock, char const **pzBlock, int *pnBlock ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_reset(pStmt); sqlite3_bind_int64(pStmt, 1, iBlock); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW ){ return SQLITE_CORRUPT; } *pnBlock = sqlite3_column_bytes(pStmt, 0); *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); if( !*pzBlock ){ return SQLITE_NOMEM; } return SQLITE_OK; } /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, ** return SQLITE_OK. If an error occurs while preparing the statement, ** return an SQLite error code. ** ** There is only ever one instance of this SQL statement compiled for ** each FTS3 table. ** ** The statement returns the following columns from the %_segdir table: ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){ return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0); } static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } return rc; } static int fts3PendingListAppendVarint( PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ sqlite3_int64 i /* Value to append to data */ ){ PendingList *p = *pp; /* Allocate or grow the PendingList as required. */ if( !p ){ p = sqlite3_malloc(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } p->nSpace = 100; p->aData = (char *)&p[1]; p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ int nNew = p->nSpace * 2; p = sqlite3_realloc(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } p->nSpace = nNew; p->aData = (char *)&p[1]; } /* Append the new serialized varint to the end of the list. */ p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); p->aData[p->nData] = '\0'; *pp = p; return SQLITE_OK; } static int fts3PendingListAppend( PendingList **pp, sqlite3_int64 iDocid, sqlite3_int64 iCol, sqlite3_int64 iPos, int *pRc ){ PendingList *p = *pp; int rc = SQLITE_OK; assert( !p || p->iLastDocid<=iDocid ); if( !p || p->iLastDocid!=iDocid ){ sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); if( p ){ assert( p->nData<p->nSpace ); assert( p->aData[p->nData]==0 ); p->nData++; } if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ goto pendinglistappend_out; } p->iLastCol = -1; p->iLastPos = 0; p->iLastDocid = iDocid; } if( iCol>0 && p->iLastCol!=iCol ){ if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) ){ goto pendinglistappend_out; } p->iLastCol = iCol; p->iLastPos = 0; } if( iCol>=0 ){ assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); p->iLastPos = iPos; } pendinglistappend_out: *pRc = rc; if( p!=*pp ){ *pp = p; return 1; } return 0; } static int fts3PendingTermsAdd(Fts3Table *p, const char *zText, int iCol){ int rc; int iStart; int iEnd; int iPos; char const *zToken; int nToken; sqlite3_tokenizer *pTokenizer = p->pTokenizer; sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; sqlite3_tokenizer_cursor *pCsr; int (*xNext)(sqlite3_tokenizer_cursor *pCursor, const char**,int*,int*,int*,int*); assert( pTokenizer && pModule ); rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr); if( rc!=SQLITE_OK ){ return rc; } pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ PendingList *pList; /* Positions cannot be negative; we use -1 as a terminator internally. ** Tokens must have a non-zero length. */ if( iPos<0 || !zToken || nToken<=0 ){ rc = SQLITE_ERROR; break; } pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); if( pList ){ p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ /* Malloc failed while inserting the new entry. This can only ** happen if there was no previous entry for this token. */ assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); sqlite3_free(pList); rc = SQLITE_NOMEM; } } if( rc==SQLITE_OK ){ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); } } pModule->xClose(pCsr); return (rc==SQLITE_DONE ? SQLITE_OK : rc); } /* ** Calling this function indicates that subsequent calls to ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ if( iDocid<=p->iPrevDocid || p->nPendingData>FTS3_MAX_PENDING_DATA ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; } void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ sqlite3_free(fts3HashData(pElem)); } fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the ** pendingTerms hash table. ** ** Argument apVal is the same as the similarly named argument passed to ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal){ int i; /* Iterator variable */ for(i=2; i<p->nColumn+2; i++){ const char *zText = (const char *)sqlite3_value_text(apVal[i]); if( zText ){ int rc = fts3PendingTermsAdd(p, zText, i-2); if( rc!=SQLITE_OK ){ return rc; } } } return SQLITE_OK; } /* ** This function is called by the xUpdate() method for an INSERT operation. ** The apVal parameter is passed a copy of the apVal argument passed by ** SQLite to the xUpdate() method. i.e: ** ** apVal[0] Not used for INSERT. ** apVal[1] rowid ** apVal[2] Left-most user-defined column ** ... ** apVal[p->nColumn+1] Right-most user-defined column ** apVal[p->nColumn+2] Hidden column with same name as table ** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) */ static int fts3InsertData( Fts3Table *p, /* Full-text table */ sqlite3_value **apVal, /* Array of values to insert */ sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ ){ int rc; /* Return code */ int i; /* Iterator variable */ sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ /* Locate the statement handle used to insert data into the %_content ** table. If no such statement has been prepared, prepare a new one. ** The SQL for this statement is: ** ** INSERT INTO %_content VALUES(?, ?, ?, ...) ** ** The statement features N '?' variables, where N is the number of user ** defined columns in the FTS3 table, plus one for the docid field. */ pContentInsert = p->aStmt[SQL_CONTENT_INSERT]; if( !pContentInsert ){ char *zVarlist; /* The "?, ?, ..." string */ char *zSql; /* The text of the INSERT statement */ /* Construct the SQL statement text. */ zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2); if( !zVarlist ){ return SQLITE_NOMEM; } zVarlist[0] = '?'; for(i=1; i<=p->nColumn; i++){ zVarlist[i*2-1] = ','; zVarlist[i*2] = '?'; } zVarlist[p->nColumn*2+1] = '\0'; zSql = sqlite3_mprintf("INSERT INTO %Q.'%q_content' VALUES(%z)", p->zDb, p->zName, zVarlist ); if( !zSql ) return SQLITE_NOMEM; /* Prepare the SQL statement. */ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pContentInsert, NULL); sqlite3_free(zSql); if( rc!=SQLITE_OK ){ return rc; } p->aStmt[SQL_CONTENT_INSERT] = pContentInsert; } /* Bind values to the prepared statement. ** ** There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. ** Which is a problem, since "rowid" and "docid" are aliases for the ** same value. For example: ** ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); ** ** In FTS3, if a non-NULL docid value is specified, it is the value ** inserted. Otherwise, the rowid value is used. */ for(i=0; i<=p->nColumn; i++){ rc = sqlite3_bind_value(pContentInsert, i+1, apVal[i+1]); if( rc!=SQLITE_OK ) return rc; } if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); if( rc!=SQLITE_OK ) return rc; } /* Execute the statement to insert the record. Set *piDocid to the ** new docid value. */ sqlite3_step(pContentInsert); rc = sqlite3_reset(pContentInsert); *piDocid = sqlite3_last_insert_rowid(p->db); return rc; } /* ** Remove all data from the FTS3 table. Clear the hash table containing ** pending terms. */ static int fts3DeleteAll(Fts3Table *p){ int rc; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ rc = fts3SqlExec(p, SQL_DELETE_ALL_CONTENT, 0); if( rc==SQLITE_OK ){ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGMENTS, 0); } if( rc==SQLITE_OK ){ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); } return rc; } /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ static int fts3DeleteTerms(Fts3Table *p, sqlite3_value **apVal){ int rc; sqlite3_stmt *pSelect; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, apVal); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; for(i=1; i<=p->nColumn; i++){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, zText, -1); if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); return rc; } } } } return sqlite3_reset(pSelect); } static int fts3SegmentMerge(Fts3Table *, int); /* ** This function allocates a new level iLevel index in the segdir table. ** Usually, indexes are allocated within a level sequentially starting ** with 0, so the allocated index is one greater than the value returned ** by: ** ** SELECT max(idx) FROM %_segdir WHERE level = :iLevel ** ** However, if there are already FTS3_MERGE_COUNT indexes at the requested ** level, they are merged into a single level (iLevel+1) segment and the ** allocated index is 0. ** ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext; /* Result of query pNextIdx */ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pNextIdx, 1, iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int64(pNextIdx, 0); } rc = sqlite3_reset(pNextIdx); } if( rc==SQLITE_OK ){ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already ** full, merge all segments in level iLevel into a single iLevel+1 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ rc = fts3SegmentMerge(p, iLevel); *piIdx = 0; }else{ *piIdx = iNext; } } return rc; } /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ static int fts3SegReaderNext(Fts3SegReader *pReader){ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ if( !pReader->aDoclist ){ pNext = pReader->aNode; }else{ pNext = &pReader->aDoclist[pReader->nDoclist]; } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ int rc; if( !pReader->pStmt ){ pReader->aNode = 0; return SQLITE_OK; } rc = sqlite3_step(pReader->pStmt); if( rc!=SQLITE_ROW ){ pReader->aNode = 0; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); pNext = pReader->aNode; } pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix+nSuffix>pReader->nTermAlloc ){ int nNew = (nPrefix+nSuffix)*2; char *zNew = sqlite3_realloc(pReader->zTerm, nNew); if( !zNew ){ return SQLITE_NOMEM; } pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); assert( pNext<&pReader->aNode[pReader->nNode] ); pReader->aDoclist = pNext; pReader->pOffsetList = 0; return SQLITE_OK; } static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ int n; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); pReader->pOffsetList = &pReader->aDoclist[n]; } /* ** ** If arguments ppOffsetList and pnOffsetList are not NULL, then ** *ppOffsetList is set to point to the first column-offset list ** in the doclist entry (i.e. immediately past the docid varint). ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: ** ** TODO: example. ** */ static void fts3SegReaderNextDocid( Fts3SegReader *pReader, char **ppOffsetList, int *pnOffsetList ){ char *p = pReader->pOffsetList; char c = 0; /* Pointer p currently points at the first byte of an offset list. The ** following two lines advance it to point one byte past the end of ** the same offset list. */ while( *p | c ) c = *p++ & 0x80; p++; /* If required, populate the output variables with a pointer to and the ** size of the previous offset-list. */ if( ppOffsetList ){ *ppOffsetList = pReader->pOffsetList; *pnOffsetList = p - pReader->pOffsetList - 1; } /* If there are no more entries in the doclist, set pOffsetList to ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and ** Fts3SegReader.pOffsetList to point to the next offset list before ** returning. */ if( p>=&pReader->aDoclist[pReader->nDoclist] ){ pReader->pOffsetList = 0; }else{ sqlite3_int64 iDelta; pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); pReader->iDocid += iDelta; } } /* ** Free all allocations associated with the iterator passed as the first ** argument. */ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ if( pReader ){ sqlite3_finalize(pReader->pStmt); sqlite3_free(pReader->zTerm); sqlite3_free(pReader); } } int sqlite3Fts3SegReaderNew( Fts3Table *p, /* Virtual table handle */ int iAge, /* Segment "age". */ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ sqlite3_int64 iEndBlock, /* Final block of segment */ const char *zRoot, /* Buffer containing root node */ int nRoot, /* Size of buffer containing root node */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ int rc = SQLITE_OK; /* Return code */ Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ if( iStartLeaf==0 ){ nExtra = nRoot; } pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } memset(pReader, 0, sizeof(Fts3SegReader)); pReader->iStartBlock = iStartLeaf; pReader->iIdx = iAge; pReader->iEndBlock = iEndBlock; if( nExtra ){ /* The entire segment is stored in the root node. */ pReader->aNode = (char *)&pReader[1]; pReader->nNode = nRoot; memcpy(pReader->aNode, zRoot, nRoot); }else{ if( !p->zSelectLeaves ){ p->zSelectLeaves = sqlite3_mprintf( "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " "ORDER BY blockid", p->zDb, p->zName ); if( !p->zSelectLeaves ){ rc = SQLITE_NOMEM; goto finished; } } rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); if( rc!=SQLITE_OK ){ goto finished; } sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); } rc = fts3SegReaderNext(pReader); finished: if( rc==SQLITE_OK ){ *ppReader = pReader; }else{ sqlite3Fts3SegReaderFree(pReader); } return rc; } /* ** The second argument to this function is expected to be a statement of ** the form: ** ** SELECT ** idx, -- col 0 ** start_block, -- col 1 ** leaves_end_block, -- col 2 ** end_block, -- col 3 ** root -- col 4 ** FROM %_segdir ... ** ** This function allocates and initializes a Fts3SegReader structure to ** iterate through the terms stored in the segment identified by the ** current row that pStmt is pointing to. ** ** If successful, the Fts3SegReader is left pointing to the first term ** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error ** code is returned. */ static int fts3SegReaderNew( Fts3Table *p, /* Virtual table handle */ sqlite3_stmt *pStmt, /* See above */ int iAge, /* Segment "age". */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ return sqlite3Fts3SegReaderNew(p, iAge, sqlite3_column_int64(pStmt, 1), sqlite3_column_int64(pStmt, 2), sqlite3_column_int64(pStmt, 3), sqlite3_column_blob(pStmt, 4), sqlite3_column_bytes(pStmt, 4), ppReader ); } /* ** Compare the two Fts3SegReader structures. Comparison is as follows: ** ** 1) EOF is greater than not EOF. ** ** 2) The current terms (if any) are compared with memcmp(). If one ** term is a prefix of another, the longer term is considered the ** larger. ** ** 3) By segment age. An older segment is considered larger. */ static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ int rc; if( pLhs->aNode && pRhs->aNode ){ int rc2 = pLhs->nTerm - pRhs->nTerm; if( rc2<0 ){ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); }else{ rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); } if( rc==0 ){ rc = rc2; } }else{ rc = (pLhs->aNode==0) - (pRhs->aNode==0); } if( rc==0 ){ rc = pRhs->iIdx - pLhs->iIdx; } assert( rc!=0 ); return rc; } static int fts3SegReaderCmp2(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); if( rc==0 ){ if( pLhs->iDocid==pRhs->iDocid ){ rc = pRhs->iIdx-pLhs->iIdx; }else{ rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; } } assert( pLhs->aNode && pRhs->aNode ); return rc; } /* ** Compare the term that the Fts3SegReader object passed as the first argument ** points to with the term specified by arguments zTerm and nTerm. ** ** If the pSeg iterator is already at EOF, return 0. Otherwise, return ** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are ** equal, or +ve if the pSeg term is greater than zTerm/nTerm. */ static int fts3SegReaderTermCmp( Fts3SegReader *pSeg, /* Segment reader object */ const char *zTerm, /* Term to compare to */ int nTerm /* Size of term zTerm in bytes */ ){ int res = 0; if( pSeg->aNode ){ if( pSeg->nTerm>nTerm ){ res = memcmp(pSeg->zTerm, zTerm, nTerm); }else{ res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); } if( res==0 ){ res = pSeg->nTerm-nTerm; } } return res; } /* ** Argument apSegment is an array of nSegment elements. It is known that ** the final (nSegment-nSuspect) members are already in sorted order ** (according to the comparison function provided). This function shuffles ** the array around until all entries are in sorted order. */ static void fts3SegReaderSort( Fts3SegReader **apSegment, /* Array to sort entries of */ int nSegment, /* Size of apSegment array */ int nSuspect, /* Unsorted entry count */ int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ ){ int i; /* Iterator variable */ assert( nSuspect<=nSegment ); if( nSuspect==nSegment ) nSuspect--; for(i=nSuspect-1; i>=0; i--){ int j; for(j=i; j<(nSegment-1); j++){ Fts3SegReader *pTmp; if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; pTmp = apSegment[j+1]; apSegment[j+1] = apSegment[j]; apSegment[j] = pTmp; } } #ifndef NDEBUG /* Check that the list really is sorted now. */ for(i=0; i<(nSuspect-1); i++){ assert( xCmp(apSegment[i], apSegment[i+1])<0 ); } #endif } /* ** Insert a record into the %_segments table. */ static int fts3WriteSegment( Fts3Table *p, sqlite3_int64 iBlock, char *z, int n ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iBlock); rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } } return rc; } /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ int iLevel, /* Value for "level" field */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); sqlite3_bind_int64(pStmt, 5, iEndBlock); rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } } return rc; } static void fts3PrefixCompress( const char *zPrev, int nPrev, const char *zNext, int nNext, int *pnPrefix ){ int n; for(n=0; n<nPrev && zPrev[n]==zNext[n]; n++); *pnPrefix = n; } /* ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( Fts3Table *p, /* Virtual table handle */ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ int isCopyTerm, /* True if zTerm/nTerm is transient */ const char *zTerm, /* Pointer to buffer containing term */ int nTerm /* Size of term in bytes */ ){ SegmentNode *pTree = *ppTree; int rc; SegmentNode *pNew; /* First try to append the term to the current node. Return early if ** this is possible. */ if( pTree ){ int nData = pTree->nData; /* Current size of node in bytes */ int nReq = nData; /* Required space after adding zTerm */ int nPrefix; /* Number of bytes of prefix compression */ int nSuffix; /* Suffix length */ fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm, &nPrefix); nSuffix = nTerm-nPrefix; nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; if( nReq<=INTERIOR_MAX || !pTree->zTerm ){ if( nReq>INTERIOR_MAX ){ /* An unusual case: this is the first term to be added to the node ** and the static node buffer (INTERIOR_MAX bytes) is not large ** enough. Use a separately malloced buffer instead This wastes ** INTERIOR_MAX bytes, but since this scenario only comes about when ** the database contain two terms that share a prefix of almost 2KB, ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); pTree->aData = (char *)sqlite3_malloc(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } } if( pTree->zTerm ){ /* There is no prefix-length field for first term in a node */ nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); } nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); pTree->nData = nData + nSuffix; pTree->nEntry++; if( isCopyTerm ){ if( pTree->nMalloc<nTerm ){ char *zNew = sqlite3_realloc(pTree->zMalloc, nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } pTree->nMalloc = nTerm*2; pTree->zMalloc = zNew; } pTree->zTerm = pTree->zMalloc; memcpy(pTree->zTerm, zTerm, nTerm); pTree->nTerm = nTerm; }else{ pTree->zTerm = (char *)zTerm; pTree->nTerm = nTerm; } return SQLITE_OK; } } /* If control flows to here, it was not possible to append zTerm to the ** current node. Create a new node (a right-sibling of the current node). ** If this is the first node in the tree, the term is added to it. ** ** Otherwise, the term is not added to the new node, it is left empty for ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + INTERIOR_MAX); if( !pNew ){ return SQLITE_NOMEM; } memset(pNew, 0, sizeof(SegmentNode)); pNew->nData = 1 + FTS3_VARINT_MAX; pNew->aData = (char *)&pNew[1]; if( pTree ){ SegmentNode *pParent = pTree->pParent; rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); if( pTree->pParent==0 ){ pTree->pParent = pParent; } pTree->pRight = pNew; pNew->pLeftmost = pTree->pLeftmost; pNew->pParent = pParent; pNew->zMalloc = pTree->zMalloc; pNew->nMalloc = pTree->nMalloc; pTree->zMalloc = 0; }else{ pNew->pLeftmost = pNew; rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); } *ppTree = pNew; return rc; } /* ** Helper function for fts3NodeWrite(). */ static int fts3TreeFinishNode( SegmentNode *pTree, int iHeight, sqlite3_int64 iLeftChild ){ int nStart; assert( iHeight>=1 && iHeight<128 ); nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); pTree->aData[nStart] = (char)iHeight; sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); return nStart; } /* ** Helper function for fts3NodeWrite(). */ static int fts3NodeWrite( Fts3Table *p, /* Virtual table handle */ SegmentNode *pTree, /* SegmentNode handle */ int iHeight, /* Height of this node in tree */ sqlite3_int64 iLeaf, /* Block id of first leaf node */ sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ char **paRoot, /* OUT: Data for root node */ int *pnRoot /* OUT: Size of root node in bytes */ ){ int rc = SQLITE_OK; if( !pTree->pParent ){ /* Root node of the tree. */ int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); *piLast = iFree-1; *pnRoot = pTree->nData - nStart; *paRoot = &pTree->aData[nStart]; }else{ SegmentNode *pIter; sqlite3_int64 iNextFree = iFree; sqlite3_int64 iNextLeaf = iLeaf; for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); int nWrite = pIter->nData - nStart; rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); iNextFree++; iNextLeaf += (pIter->nEntry+1); } if( rc==SQLITE_OK ){ assert( iNextLeaf==iFree ); rc = fts3NodeWrite( p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot ); } } return rc; } /* ** Free all memory allocations associated with the tree pTree. */ static void fts3NodeFree(SegmentNode *pTree){ if( pTree ){ SegmentNode *p = pTree->pLeftmost; fts3NodeFree(p->pParent); while( p ){ SegmentNode *pRight = p->pRight; if( p->aData!=(char *)&p[1] ){ sqlite3_free(p->aData); } assert( pRight==0 || p->zMalloc==0 ); sqlite3_free(p->zMalloc); sqlite3_free(p); p = pRight; } } } static int fts3SegWriterAdd( Fts3Table *p, /* Virtual table handle */ SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ int isCopyTerm, /* True if buffer zTerm must be copied */ const char *zTerm, /* Pointer to buffer containing term */ int nTerm, /* Size of term in bytes */ const char *aDoclist, /* Pointer to buffer containing doclist */ int nDoclist /* Size of doclist in bytes */ ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ int nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; if( !pWriter ){ int rc; sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ pWriter->aData = (char *)sqlite3_malloc(LEAF_MAX); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = LEAF_MAX; /* Find the next free blockid in the %_segments table */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; if( SQLITE_ROW==sqlite3_step(pStmt) ){ pWriter->iFree = sqlite3_column_int64(pStmt, 0); pWriter->iFirst = pWriter->iFree; } rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK ) return rc; } nData = pWriter->nData; fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm, &nPrefix); nSuffix = nTerm-nPrefix; /* Figure out how many bytes are required by this new entry */ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ nSuffix + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ if( nData>0 && nData+nReq>LEAF_MAX ){ int rc; /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; /* Add the current term to the interior node tree. The term added to ** the interior tree must: ** ** a) be greater than the largest term on the leaf node just written ** to the database (still available in pWriter->zTerm), and ** ** b) be less than or equal to the term about to be added to the new ** leaf node (zTerm/nTerm). ** ** In other words, it must be the prefix of zTerm 1 byte longer than ** the common prefix (if any) of zTerm and pWriter->zTerm. */ assert( nPrefix<nTerm ); rc = fts3NodeAddTerm(p, &pWriter->pTree, isCopyTerm, zTerm, nPrefix+1); if( rc!=SQLITE_OK ) return rc; nData = 0; pWriter->nTerm = 0; nPrefix = 0; nSuffix = nTerm; nReq = 1 + /* varint containing prefix size */ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ char *aNew = sqlite3_realloc(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; } assert( nData+nReq<=pWriter->nSize ); /* Append the prefix-compressed term and doclist to the buffer. */ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); nData += nSuffix; nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); memcpy(&pWriter->aData[nData], aDoclist, nDoclist); pWriter->nData = nData + nDoclist; /* Save the current term so that it can be used to prefix-compress the next. ** If the isCopyTerm parameter is true, then the buffer pointed to by ** zTerm is transient, so take a copy of the term data. Otherwise, just ** store a copy of the pointer. */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } pWriter->nMalloc = nTerm*2; pWriter->zMalloc = zNew; pWriter->zTerm = zNew; } assert( pWriter->zTerm==pWriter->zMalloc ); memcpy(pWriter->zTerm, zTerm, nTerm); }else{ pWriter->zTerm = (char *)zTerm; } pWriter->nTerm = nTerm; return SQLITE_OK; } static int fts3SegWriterFlush( Fts3Table *p, SegmentWriter *pWriter, int iLevel, int iIdx ){ int rc; if( pWriter->pTree ){ sqlite3_int64 iLast; char *zRoot; int nRoot; sqlite3_int64 iLastLeaf = pWriter->iFree; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ rc = fts3WriteSegdir( p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } return rc; } static void fts3SegWriterFree(SegmentWriter *pWriter){ if( pWriter ){ sqlite3_free(pWriter->aData); sqlite3_free(pWriter->zMalloc); fts3NodeFree(pWriter->pTree); sqlite3_free(pWriter); } } /* ** The first value in the apVal[] array is assumed to contain an integer. ** This function tests if there exist any documents with docid values that ** are different from that integer. i.e. if deleting the document with docid ** apVal[0] would mean the FTS3 table were empty. ** ** If successful, *pisEmpty is set to true if the table is empty except for ** document apVal[0], or false otherwise, and SQLITE_OK is returned. If an ** error occurs, an SQLite error code is returned. */ static int fts3IsEmpty(Fts3Table *p, sqlite3_value **apVal, int *pisEmpty){ sqlite3_stmt *pStmt; int rc; rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, apVal); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pisEmpty = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } return rc; } static int fts3SegmentCount(Fts3Table *p, int iLevel, int *pnSegment){ sqlite3_stmt *pStmt; int rc; assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int(pStmt, 1, iLevel); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnSegment = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ sqlite3_stmt *pStmt; int rc; rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnSegment = sqlite3_column_int(pStmt, 0); *pnMax = sqlite3_column_int(pStmt, 1); } return sqlite3_reset(pStmt); } static int fts3DeleteSegdir( Fts3Table *p, int iLevel, Fts3SegReader **apSegment, int nReader ){ int rc; /* Return Code */ int i; /* Iterator variable */ sqlite3_stmt *pDelete; /* SQL statement to delete rows */ rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); for(i=0; rc==SQLITE_OK && i<nReader; i++){ Fts3SegReader *pSegment = apSegment[i]; if( pSegment->iStartBlock ){ sqlite3_bind_int64(pDelete, 1, pSegment->iStartBlock); sqlite3_bind_int64(pDelete, 2, pSegment->iEndBlock); sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } if( iLevel>=0 ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } }else{ rc = fts3SqlExec(p, SQL_DELETE_ALL_SEGDIR, 0); } return rc; } static void fts3ColumnFilter(int iCol, char **ppList, int *pnList){ char *pList = *ppList; int nList = *pnList; char *pEnd = &pList[nList]; int iCurrent = 0; char *p = pList; assert( iCol>=0 ); while( 1 ){ char c = 0; while( p<pEnd && (c | *p)&0xFE ) c = *p++ & 0x80; if( iCol==iCurrent ){ nList = (p - pList); break; } nList -= (p - pList); pList = p; if( nList==0 ){ break; } p = &pList[1]; p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } static int fts3MergeCallback( Fts3Table *p, void *pContext, char *zTerm, int nTerm, char *aDoclist, int nDoclist ){ SegmentWriter **ppW = (SegmentWriter **)pContext; return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist); } int sqlite3Fts3SegReaderIterate( Fts3Table *p, /* Virtual table handle */ Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */ int nSegment, /* Size of apSegment array */ Fts3SegFilter *pFilter, /* Restrictions on range of iteration */ int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */ void *pContext /* Callback context (2nd argument) */ ){ int i; /* Iterator variable */ char *aBuffer = 0; /* Buffer to merge doclists in */ int nAlloc = 0; /* Allocated size of aBuffer buffer */ int rc = SQLITE_OK; /* Return code */ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ if( pFilter->zTerm ){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; for(i=0; i<nSegment; i++){ Fts3SegReader *pSeg = apSegment[i]; while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){ rc = fts3SegReaderNext(pSeg); if( rc!=SQLITE_OK ) goto finished; } } } fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); while( apSegment[0]->aNode ){ int nTerm = apSegment[0]->nTerm; char *zTerm = apSegment[0]->zTerm; int nMerge = 1; /* If this is a prefix-search, and if the term that apSegment[0] points ** to does not share a suffix with pFilter->zTerm/nTerm, then all ** required callbacks have been made. In this case exit early. ** ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ if( pFilter->zTerm ){ if( nTerm<pFilter->nTerm || (!isPrefix && nTerm>pFilter->nTerm) || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) ){ goto finished; } } while( nMerge<nSegment && apSegment[nMerge]->aNode && apSegment[nMerge]->nTerm==nTerm && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm) ){ nMerge++; } if( nMerge==1 && !isIgnoreEmpty && !isColFilter && isRequirePos ){ Fts3SegReader *p0 = apSegment[0]; rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); if( rc!=SQLITE_OK ) goto finished; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the linked list ** of Fts3SegReader objects is the same. The doclists must be merged ** and a single term added to the new segment. */ for(i=0; i<nMerge; i++){ fts3SegReaderFirstDocid(apSegment[i]); } fts3SegReaderSort(apSegment, nMerge, nMerge, fts3SegReaderCmp2); while( apSegment[0]->pOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; fts3SegReaderNextDocid(apSegment[0], &pList, &nList); j = 1; while( j<nMerge && apSegment[j]->pOffsetList && apSegment[j]->iDocid==iDocid ){ fts3SegReaderNextDocid(apSegment[j], 0, 0); j++; } if( isColFilter ){ fts3ColumnFilter(pFilter->iCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); if( nDoclist+nByte>nAlloc ){ char *aNew; nAlloc = nDoclist+nByte*2; aNew = sqlite3_realloc(aBuffer, nAlloc); if( !aNew ){ rc = SQLITE_NOMEM; goto finished; } aBuffer = aNew; } nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev); iPrev = iDocid; if( isRequirePos ){ memcpy(&aBuffer[nDoclist], pList, nList); nDoclist += nList; aBuffer[nDoclist++] = '\0'; } } fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderCmp2); } if( nDoclist>0 ){ rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist); if( rc!=SQLITE_OK ) goto finished; } } /* If there is a term specified to filter on, and this is not a prefix ** search, return now. The callback that corresponds to the required ** term (if such a term exists in the index) has already been made. */ if( pFilter->zTerm && !isPrefix ){ goto finished; } for(i=0; i<nMerge; i++){ rc = fts3SegReaderNext(apSegment[i]); if( rc!=SQLITE_OK ) goto finished; } fts3SegReaderSort(apSegment, nSegment, nMerge, fts3SegReaderCmp); } finished: sqlite3_free(aBuffer); return rc; } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ int i; /* Iterator variable */ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel; /* Level to create new segment at */ sqlite3_stmt *pStmt; SegmentWriter *pWriter = 0; int nSegment = 0; /* Number of segments being merged */ Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ Fts3SegFilter filter; /* Segment term filter condition */ if( iLevel<0 ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ iIdx = 0; rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel); if( nSegment==1 ){ return SQLITE_DONE; } }else{ /* This call is to merge all segments at level iLevel. Find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ iNewLevel = iLevel+1; rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); if( rc!=SQLITE_OK ) return rc; rc = fts3SegmentCount(p, iLevel, &nSegment); } if( rc!=SQLITE_OK ) return rc; assert( nSegment>0 ); assert( iNewLevel>=0 ); /* Allocate space for an array of pointers to segment iterators. */ apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); if( !apSegment ){ return SQLITE_NOMEM; } memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); /* Allocate a Fts3SegReader structure for each segment being merged. A ** Fts3SegReader stores the state data required to iterate through all ** entries on all leaves of a single segment. */ assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL); rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0); if( rc!=SQLITE_OK ) goto finished; sqlite3_bind_int(pStmt, 1, iLevel); for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){ rc = fts3SegReaderNew(p, pStmt, i, &apSegment[i]); if( rc!=SQLITE_OK ){ goto finished; } } rc = sqlite3_reset(pStmt); pStmt = 0; if( rc!=SQLITE_OK ) goto finished; memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, fts3MergeCallback, (void *)&pWriter ); if( rc!=SQLITE_OK ) goto finished; rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment); if( rc==SQLITE_OK ){ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); } finished: fts3SegWriterFree(pWriter); if( apSegment ){ for(i=0; i<nSegment; i++){ sqlite3Fts3SegReaderFree(apSegment[i]); } sqlite3_free(apSegment); } sqlite3_reset(pStmt); return rc; } /* ** This is a comparison function used as a qsort() callback when sorting ** an array of pending terms by term. This occurs as part of flushing ** the contents of the pending-terms hash table to the database. */ static int qsortCompare(const void *lhs, const void *rhs){ char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); int n = (n1<n2 ? n1 : n2); int c = memcmp(z1, z2, n); if( c==0 ){ c = n1 - n2; } return c; } /* ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ Fts3HashElem *pElem; int idx, rc, i; Fts3HashElem **apElem; /* Array of pointers to hash elements */ int nElem; /* Number of terms in new segment */ SegmentWriter *pWriter = 0; /* Used to write the segment */ /* Find the number of terms that will make up the new segment. If there ** are no terms, return early (do not bother to write an empty segment). */ nElem = fts3HashCount(&p->pendingTerms); if( nElem==0 ){ assert( p->nPendingData==0 ); return SQLITE_OK; } /* Determine the next index at level 0, merging as necessary. */ rc = fts3AllocateSegdirIdx(p, 0, &idx); if( rc!=SQLITE_OK ){ return rc; } apElem = sqlite3_malloc(nElem*sizeof(Fts3HashElem *)); if( !apElem ){ return SQLITE_NOMEM; } i = 0; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ apElem[i++] = pElem; } assert( i==nElem ); /* TODO(shess) Should we allow user-defined collation sequences, ** here? I think we only need that once we support prefix searches. ** Also, should we be using qsort()? */ if( nElem>1 ){ qsort(apElem, nElem, sizeof(Fts3HashElem *), qsortCompare); } /* Write the segment tree into the database. */ for(i=0; rc==SQLITE_OK && i<nElem; i++){ const char *z = fts3HashKey(apElem[i]); int n = fts3HashKeysize(apElem[i]); PendingList *pList = fts3HashData(apElem[i]); rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1); } if( rc==SQLITE_OK ){ rc = fts3SegWriterFlush(p, pWriter, 0, idx); } /* Free all allocated resources before returning */ fts3SegWriterFree(pWriter); sqlite3_free(apElem); sqlite3Fts3PendingTermsClear(p); return rc; } /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. */ int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove; /* Rowid removed by UPDATE or DELETE */ /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ int isEmpty; rc = fts3IsEmpty(p, apVal, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any ** data in the pendingTerms hash table. */ rc = fts3DeleteAll(p); }else{ isRemove = 1; iRemove = sqlite3_value_int64(apVal[0]); rc = fts3PendingTermsDocid(p, iRemove); if( rc==SQLITE_OK ){ rc = fts3DeleteTerms(p, apVal); if( rc==SQLITE_OK ){ rc = fts3SqlExec(p, SQL_DELETE_CONTENT, apVal); } } } } } /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ rc = fts3InsertData(p, apVal, pRowid); if( rc==SQLITE_OK && (!isRemove || *pRowid!=iRemove) ){ rc = fts3PendingTermsDocid(p, *pRowid); } if( rc==SQLITE_OK ){ rc = fts3InsertTerms(p, apVal); } } return rc; } /* ** Flush any data in the pending-terms hash table to disk. If successful, ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK ){ rc = fts3SegmentMerge(p, -1); } return rc; } |
Changes to main.mk.
︙ | ︙ | |||
50 51 52 53 54 55 56 | # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ | | | 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \ pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ |
︙ | ︙ | |||
179 180 181 182 183 184 185 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3_expr.c \ | < | > | 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | $(TOP)/ext/fts2/fts2_tokenizer.h \ $(TOP)/ext/fts2/fts2_tokenizer.c \ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_tokenizer1.c \ $(TOP)/ext/fts3/fts3_write.c SRC += \ $(TOP)/ext/icu/sqliteicu.h \ $(TOP)/ext/icu/icu.c SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c |
︙ | ︙ | |||
292 293 294 295 296 297 298 | $(TOP)/ext/fts1/fts1_tokenizer.h EXTHDR += \ $(TOP)/ext/fts2/fts2.h \ $(TOP)/ext/fts2/fts2_hash.h \ $(TOP)/ext/fts2/fts2_tokenizer.h EXTHDR += \ $(TOP)/ext/fts3/fts3.h \ | | | 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 | $(TOP)/ext/fts1/fts1_tokenizer.h EXTHDR += \ $(TOP)/ext/fts2/fts2.h \ $(TOP)/ext/fts2/fts2_hash.h \ $(TOP)/ext/fts2/fts2_tokenizer.h EXTHDR += \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_tokenizer.h EXTHDR += \ $(TOP)/ext/rtree/rtree.h EXTHDR += \ $(TOP)/ext/icu/sqliteicu.h |
︙ | ︙ | |||
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c # Rules for building test programs and for running tests # | > > > > > > | 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c fts3_snippet.o: $(TOP)/ext/fts3/fts3_snippet.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_snippet.c fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c # Rules for building test programs and for running tests # |
︙ | ︙ |
Changes to src/test_hexio.c.
︙ | ︙ | |||
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 | sqlite3TestBinToHex(z,nOut); Tcl_AppendResult(interp, (char*)z, 0); sqlite3_free(z); #endif return TCL_OK; } /* ** Register commands with the TCL interpreter. */ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ static struct { char *zName; Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "hexio_read", hexio_read }, { "hexio_write", hexio_write }, { "hexio_get_int", hexio_get_int }, { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, }; int i; for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, 0, 0); } return TCL_OK; } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 | sqlite3TestBinToHex(z,nOut); Tcl_AppendResult(interp, (char*)z, 0); sqlite3_free(z); #endif return TCL_OK; } static int getFts3Varint(const char *p, sqlite_int64 *v){ const unsigned char *q = (const unsigned char *) p; sqlite_uint64 x = 0, y = 1; while( (*q & 0x80) == 0x80 ){ x += y * (*q++ & 0x7f); y <<= 7; } x += y * (*q++); *v = (sqlite_int64) x; return (int) (q - (unsigned char *)p); } /* ** USAGE: read_varint BLOB VARNAME ** ** Read a varint from the start of BLOB. Set variable VARNAME to contain ** the interpreted value. Return the number of bytes of BLOB consumed. */ static int read_varint( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int nBlob; unsigned char *zBlob; sqlite3_int64 iVal; int nVal; if( objc!=3 ){ Tcl_WrongNumArgs(interp, 1, objv, "BLOB VARNAME"); return TCL_ERROR; } zBlob = Tcl_GetByteArrayFromObj(objv[1], &nBlob); nVal = getFts3Varint(zBlob, (sqlite3_uint64 *)(&iVal)); Tcl_ObjSetVar2(interp, objv[2], 0, Tcl_NewWideIntObj(iVal), 0); Tcl_SetObjResult(interp, Tcl_NewIntObj(nVal)); return TCL_OK; } /* ** Register commands with the TCL interpreter. */ int Sqlitetest_hexio_Init(Tcl_Interp *interp){ static struct { char *zName; Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "hexio_read", hexio_read }, { "hexio_write", hexio_write }, { "hexio_get_int", hexio_get_int }, { "hexio_render_int16", hexio_render_int16 }, { "hexio_render_int32", hexio_render_int32 }, { "utf8_to_utf8", utf8_to_utf8 }, { "read_varint", read_varint }, }; int i; for(i=0; i<sizeof(aObjCmd)/sizeof(aObjCmd[0]); i++){ Tcl_CreateObjCommand(interp, aObjCmd[i].zName, aObjCmd[i].xProc, 0, 0); } return TCL_OK; } |
Changes to test/fts3.test.
︙ | ︙ | |||
38 39 40 41 42 43 44 45 46 47 48 49 50 51 | } rename finish_test really_finish_test proc finish_test {} {} set ISQUICK 1 set EXCLUDE { fts3.test } # Files to include in the test. If this list is empty then everything # that is not in the EXCLUDE list is run. # set INCLUDE { } | > | 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 | } rename finish_test really_finish_test proc finish_test {} {} set ISQUICK 1 set EXCLUDE { fts3.test fts3malloc.test } # Files to include in the test. If this list is empty then everything # that is not in the EXCLUDE list is run. # set INCLUDE { } |
︙ | ︙ |
Added test/fts3_common.tcl.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 | # 2009 November 04 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains common code used the fts3 tests. At one point # equivalent functionality was implemented in C code. But it is easier # to use Tcl. # #------------------------------------------------------------------------- # USAGE: fts3_integrity_check TBL # # This proc is used to verify that the full-text index is consistent with # the contents of the fts3 table. In other words, it checks that the # data in the %_contents table matches that in the %_segdir and %_segments # tables. # # This is not an efficient procedure. It uses a lot of memory and a lot # of CPU. But it is better than not checking at all. # # The procedure is: # # 1) Read the entire full-text index from the %_segdir and %_segments # tables into memory. For each entry in the index, the following is # done: # # set C($iDocid,$iCol,$iPosition) $zTerm # # 2) Iterate through each column of each row of the %_content table. # Tokenize all documents, and check that for each token there is # a corresponding entry in the $C array. After checking a token, # [unset] the $C array entry. # # 3) Check that array $C is now empty. # # proc fts3_integrity_check {tbl} { fts3_read2 $tbl 1 A foreach zTerm [array names A] { foreach doclist $A($zTerm) { set docid 0 while {[string length $doclist]>0} { set iCol 0 set iPos 0 set lPos [list] set lCol [list] # First varint of a doclist-entry is the docid. Delta-compressed # with respect to the docid of the previous entry. # incr docid [gobble_varint doclist] if {[info exists D($zTerm,$docid)]} { while {[set iDelta [gobble_varint doclist]] != 0} {} continue } set D($zTerm,$docid) 1 # Gobble varints until the 0x00 that terminates the doclist-entry # is found. while {[set iDelta [gobble_varint doclist]] > 0} { if {$iDelta == 1} { set iCol [gobble_varint doclist] set iPos 0 } else { incr iPos $iDelta incr iPos -2 set C($docid,$iCol,$iPos) $zTerm } } } } } foreach key [array names C] { #puts "$key -> $C($key)" } db eval "SELECT * FROM ${tbl}_content" E { set iCol 0 set iDoc $E(docid) foreach col [lrange $E(*) 1 end] { set c $E($col) set sql {SELECT fts3_tokenizer_test('simple', $c)} foreach {pos term dummy} [db one $sql] { if {![info exists C($iDoc,$iCol,$pos)]} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index is missing" lappend errors $es } else { if {$C($iDoc,$iCol,$pos) != "$term"} { set es "Error at docid=$iDoc col=$iCol pos=$pos. Index " append es "has \"$C($iDoc,$iCol,$pos)\", document has \"$term\"" lappend errors $es } unset C($iDoc,$iCol,$pos) } } incr iCol } } foreach c [array names C] { lappend errors "Bad index entry: $c -> $C($c)" } if {[info exists errors]} { return [join $errors "\n"] } return "ok" } # USAGE: fts3_terms TBL WHERE # # Argument TBL must be the name of an FTS3 table. Argument WHERE is an # SQL expression that will be used as the WHERE clause when scanning # the %_segdir table. As in the following query: # # "SELECT * FROM ${TBL}_segdir WHERE ${WHERE}" # # This function returns a list of all terms present in the segments # selected by the statement above. # proc fts3_terms {tbl where} { fts3_read $tbl $where a return [lsort [array names a]] } # USAGE: fts3_doclist TBL TERM WHERE # # Argument TBL must be the name of an FTS3 table. TERM is a term that may # or may not be present in the table. Argument WHERE is used to select a # subset of the b-tree segments in the associated full-text index as # described above for [fts3_terms]. # # This function returns the results of merging the doclists associated # with TERM in the selected segments. Each doclist is an element of the # returned list. Each doclist is formatted as follows: # # [$docid ?$col[$off1 $off2...]?...] # # The formatting is odd for a Tcl command in order to be compatible with # the original C-language implementation. If argument WHERE is "1", then # any empty doclists are omitted from the returned list. # proc fts3_doclist {tbl term where} { fts3_read $tbl $where a foreach doclist $a($term) { set docid 0 while {[string length $doclist]>0} { set iCol 0 set iPos 0 set lPos [list] set lCol [list] incr docid [gobble_varint doclist] while {[set iDelta [gobble_varint doclist]] > 0} { if {$iDelta == 1} { lappend lCol [list $iCol $lPos] set iPos 0 set lPos [list] set iCol [gobble_varint doclist] } else { incr iPos $iDelta incr iPos -2 lappend lPos $iPos } } if {[llength $lPos]>0} { lappend lCol [list $iCol $lPos] } if {$where != "1" || [llength $lCol]>0} { set ret($docid) $lCol } else { unset -nocomplain ret($docid) } } } set lDoc [list] foreach docid [lsort -integer [array names ret]] { set lCol [list] set cols "" foreach col $ret($docid) { foreach {iCol lPos} $col {} append cols " $iCol\[[join $lPos { }]\]" } lappend lDoc "\[${docid}${cols}\]" } join $lDoc " " } ########################################################################### proc gobble_varint {varname} { upvar $varname blob set n [read_varint $blob ret] set blob [string range $blob $n end] return $ret } proc gobble_string {varname nLength} { upvar $varname blob set ret [string range $blob 0 [expr $nLength-1]] set blob [string range $blob $nLength end] return $ret } # The argument is a blob of data representing an FTS3 segment leaf. # Return a list consisting of alternating terms (strings) and doclists # (blobs of data). # proc fts3_readleaf {blob} { set zPrev "" set terms [list] while {[string length $blob] > 0} { set nPrefix [gobble_varint blob] set nSuffix [gobble_varint blob] set zTerm [string range $zPrev 0 [expr $nPrefix-1]] append zTerm [gobble_string blob $nSuffix] set doclist [gobble_string blob [gobble_varint blob]] lappend terms $zTerm $doclist set zPrev $zTerm } return $terms } proc fts3_read2 {tbl where varname} { upvar $varname a array unset a db eval " SELECT start_block, leaves_end_block, root FROM ${tbl}_segdir WHERE $where ORDER BY level ASC, idx DESC " { if {$start_block == 0} { foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } } else { db eval " SELECT block FROM ${tbl}_segments WHERE blockid>=$start_block AND blockid<=$leaves_end_block ORDER BY blockid " { foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } } } } } proc fts3_read {tbl where varname} { upvar $varname a array unset a db eval " SELECT start_block, leaves_end_block, root FROM ${tbl}_segdir WHERE $where ORDER BY level DESC, idx ASC " { if {$start_block == 0} { foreach {t d} [fts3_readleaf $root] { lappend a($t) $d } } else { db eval " SELECT block FROM ${tbl}_segments WHERE blockid>=$start_block AND blockid<$leaves_end_block ORDER BY blockid " { foreach {t d} [fts3_readleaf $block] { lappend a($t) $d } } } } } |
Changes to test/fts3aa.test.
︙ | ︙ | |||
142 143 144 145 146 147 148 149 150 151 152 153 154 155 | do_test fts3aa-3.2 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one -two'} } {1 5 9 13 17 21 25 29} do_test fts3aa-3.3 { execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'} } {1 5 9 13 17 21 25 29} do_test fts3aa-4.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'} } {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31} do_test fts3aa-4.2 { execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two" OR three'} } {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31} do_test fts3aa-4.3 { | > | 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | do_test fts3aa-3.2 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one -two'} } {1 5 9 13 17 21 25 29} do_test fts3aa-3.3 { execsql {SELECT rowid FROM t1 WHERE content MATCH '-two one'} } {1 5 9 13 17 21 25 29} breakpoint do_test fts3aa-4.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one OR two'} } {1 2 3 5 6 7 9 10 11 13 14 15 17 18 19 21 22 23 25 26 27 29 30 31} do_test fts3aa-4.2 { execsql {SELECT rowid FROM t1 WHERE content MATCH '"one two" OR three'} } {3 4 5 6 7 11 12 13 14 15 19 20 21 22 23 27 28 29 30 31} do_test fts3aa-4.3 { |
︙ | ︙ | |||
191 192 193 194 195 196 197 198 199 200 201 202 | } {{four five}} do_test fts3aa-6.2 { execsql {INSERT INTO t1(rowid, content) VALUES(-1, 'three four')} } {} do_test fts3aa-6.3 { execsql {SELECT content FROM t1 WHERE rowid = -1} } {{three four}} do_test fts3aa-6.4 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'four'} } {-1 0 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31} finish_test | > | 192 193 194 195 196 197 198 199 200 201 202 203 204 | } {{four five}} do_test fts3aa-6.2 { execsql {INSERT INTO t1(rowid, content) VALUES(-1, 'three four')} } {} do_test fts3aa-6.3 { execsql {SELECT content FROM t1 WHERE rowid = -1} } {{three four}} breakpoint do_test fts3aa-6.4 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'four'} } {-1 0 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31} finish_test |
Changes to test/fts3ab.test.
︙ | ︙ | |||
111 112 113 114 115 116 117 118 119 120 121 122 123 124 | CREATE VIRTUAL TABLE t4 USING fts3([norm],'plusone',"invert"); } for {set i 1} {$i<=15} {incr i} { set vset [list [wordset $i] [wordset [expr {$i+1}]] [wordset [expr {~$i}]]] db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);" } do_test fts3ab-4.1 { execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'} } {1 3 5 7 9 11 13 15} do_test fts3ab-4.2 { execsql {SELECT rowid FROM t4 WHERE norm MATCH 'one'} } {1 3 5 7 9 11 13 15} do_test fts3ab-4.3 { | > | 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | CREATE VIRTUAL TABLE t4 USING fts3([norm],'plusone',"invert"); } for {set i 1} {$i<=15} {incr i} { set vset [list [wordset $i] [wordset [expr {$i+1}]] [wordset [expr {~$i}]]] db eval "INSERT INTO t4(norm,plusone,invert) VALUES([join $vset ,]);" } breakpoint do_test fts3ab-4.1 { execsql {SELECT rowid FROM t4 WHERE t4 MATCH 'norm:one'} } {1 3 5 7 9 11 13 15} do_test fts3ab-4.2 { execsql {SELECT rowid FROM t4 WHERE norm MATCH 'one'} } {1 3 5 7 9 11 13 15} do_test fts3ab-4.3 { |
︙ | ︙ |
Changes to test/fts3ae.test.
︙ | ︙ | |||
54 55 56 57 58 59 60 | INSERT INTO t1 (rowid, content) VALUES(20, 'three five'); INSERT INTO t1 (rowid, content) VALUES(21, 'one three five'); INSERT INTO t1 (rowid, content) VALUES(22, 'two three five'); DELETE FROM t1 WHERE rowid = 19; DELETE FROM t1 WHERE rowid = 22; } | | | 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | INSERT INTO t1 (rowid, content) VALUES(20, 'three five'); INSERT INTO t1 (rowid, content) VALUES(21, 'one three five'); INSERT INTO t1 (rowid, content) VALUES(22, 'two three five'); DELETE FROM t1 WHERE rowid = 19; DELETE FROM t1 WHERE rowid = 22; } do_test fts3ae-1.1 { execsql {SELECT COUNT(*) FROM t1} } {14} do_test fts3ae-2.1 { execsql {SELECT rowid FROM t1 WHERE content MATCH 'one'} } {3 5 9 11 15 17 21} |
︙ | ︙ |
Changes to test/fts3ag.test.
︙ | ︙ | |||
74 75 76 77 78 79 80 81 82 83 84 85 86 87 | do_test fts3ag-1.10 { catchsql {SELECT rowid FROM t1 WHERE t1 MATCH '-this -something'} } {1 {SQL logic error or missing database}} # Test that docListOrMerge() correctly handles reaching the end of one # doclist before it reaches the end of the other. do_test fts3ag-1.11 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR also'} } {1 2} do_test fts3ag-1.12 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'also OR this'} } {1 2} # Empty left and right in docListOrMerge(). Each term matches neither | > | 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | do_test fts3ag-1.10 { catchsql {SELECT rowid FROM t1 WHERE t1 MATCH '-this -something'} } {1 {SQL logic error or missing database}} # Test that docListOrMerge() correctly handles reaching the end of one # doclist before it reaches the end of the other. do_test fts3ag-1.11 { breakpoint execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'this OR also'} } {1 2} do_test fts3ag-1.12 { execsql {SELECT rowid FROM t1 WHERE t1 MATCH 'also OR this'} } {1 2} # Empty left and right in docListOrMerge(). Each term matches neither |
︙ | ︙ |
Changes to test/fts3an.test.
︙ | ︙ | |||
165 166 167 168 169 170 171 | db eval { BEGIN; CREATE VIRTUAL TABLE t3 USING fts3(c); INSERT INTO t3(rowid, c) VALUES(1, $text); INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row'); } | | | 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 | db eval { BEGIN; CREATE VIRTUAL TABLE t3 USING fts3(c); INSERT INTO t3(rowid, c) VALUES(1, $text); INSERT INTO t3(rowid, c) VALUES(2, 'Another lovely row'); } for {set i 0} {$i<68} {incr i} { db eval {INSERT INTO t3(rowid, c) VALUES(3+$i, $bigtext)} lappend ret 192 } db eval {COMMIT;} # Test that we get the expected number of hits. do_test fts3an-3.1 { |
︙ | ︙ |
Changes to test/fts3c.test.
︙ | ︙ | |||
8 9 10 11 12 13 14 | # May you share freely, never taking more than you give. # #************************************************************************* # This file exercises some new testing functions in the FTS3 module, # and then uses them to do some basic tests that FTS3 is internally # working as expected. # | < < > < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | < < | < < | < < | < < < < < < | < | < < | < < < | < < < < | 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | # May you share freely, never taking more than you give. # #************************************************************************* # This file exercises some new testing functions in the FTS3 module, # and then uses them to do some basic tests that FTS3 is internally # working as expected. # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { set where "level = $level AND idx = $index" do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { set where "level = $level AND idx = $index" do_test $test [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { do_test $test [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* # Test the segments resulting from straight-forward inserts. db eval { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING fts3(c); |
︙ | ︙ |
Changes to test/fts3d.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 | # 2008 June 26 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The focus # of this script is testing the FTS3 module's optimize() function. # | < < > < < < < < < < < < < < < < < < < < | < < | < < | < < | < < < < < < | < | < < | < < < | < < < < | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | # 2008 June 26 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The focus # of this script is testing the FTS3 module's optimize() function. # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/fts3_common.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test return } #************************************************************************* # Utility function to check for the expected terms in the segment # level/index. _all version does same but for entire index. proc check_terms {test level index terms} { set where "level = $level AND idx = $index" do_test $test.terms [list fts3_terms t1 $where] $terms } proc check_terms_all {test terms} { do_test $test.terms [list fts3_terms t1 1] $terms } # Utility function to check for the expected doclist for the term in # segment level/index. _all version does same for entire index. proc check_doclist {test level index term doclist} { set where "level = $level AND idx = $index" do_test $test.doclist [list fts3_doclist t1 $term $where] $doclist } proc check_doclist_all {test term doclist} { do_test $test.doclist [list fts3_doclist t1 $term 1] $doclist } #************************************************************************* # Test results when all rows are deleted and one is added back. # Previously older segments would continue to exist, but now the index # should be dropped when the table is empty. The results should look # exactly like we never added the earlier rows in the first place. |
︙ | ︙ | |||
289 290 291 292 293 294 295 296 297 298 299 300 301 302 | check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. do_test fts3d-4.5 { execsql { SELECT OPTIMIZE(t1) FROM t1 LIMIT 1; SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {{Index optimized} 1 0} | > | 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 | check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. breakpoint do_test fts3d-4.5 { execsql { SELECT OPTIMIZE(t1) FROM t1 LIMIT 1; SELECT level, idx FROM t1_segdir ORDER BY level, idx; } } {{Index optimized} 1 0} |
︙ | ︙ |
Added test/fts3malloc.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 | # 2009 October 22 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests to verify that malloc() errors that occur # within the FTS3 module code are handled correctly. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !fts3 { finish_test ; return } source $testdir/malloc_common.tcl source $testdir/fts3_common.tcl # Ensure the lookaside buffer is disabled for these tests. # sqlite3 db test.db sqlite3_db_config_lookaside db 0 0 0 set sqlite_fts3_enable_parentheses 1 set DO_MALLOC_TEST 1 # Test organization: # # fts3_malloc-1.*: Test CREATE and DROP table statements. # # #------------------------------------------------------------------------- # This proc is used to test a single SELECT statement. Parameter $name is # passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter # $sql is passed the text of the SELECT statement. Parameter $result is # set to the expected output if the SELECT statement is successfully # executed using [db eval]. # # Example: # # do_select_test testcase-1.1 "SELECT 1+1, 1+2" {1 2} # # If global variable DO_MALLOC_TEST is set to a non-zero value, or if # it is not defined at all, then OOM testing is performed on the SELECT # statement. Each OOM test case is said to pass if either (a) executing # the SELECT statement succeeds and the results match those specified # by parameter $result, or (b) TCL throws an "out of memory" error. # # If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement # is executed just once. In this case the test case passes if the results # match the expected results passed via parameter $result. # proc do_passive_test {name sql catchres} { if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } if {$::DO_MALLOC_TEST} { set answers [list {1 {out of memory}} $catchres] set modes [list 100000 transient 1 persistent] } else { set answers [list $catchres] set modes [list 0 nofail] } set str [join $answers " OR "] foreach {nRepeat zName} $modes { for {set iFail 1} 1 {incr iFail} { if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} set res [catchsql $sql] if {[lsearch $answers $res]>=0} { set res $str } do_test $name.$zName.$iFail [list set {} $res] $str set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] if {$nFail==0} break } } } proc do_select_test {name sql result} { do_passive_test $name $sql [list 0 $result] } proc do_error_test {name sql error} { do_passive_test $name $sql [list 1 $error] } #------------------------------------------------------------------------- # Test a single write to the database. In this case a "write" is a # DELETE, UPDATE or INSERT statement. # # If OOM testing is performed, there are several acceptable outcomes: # # 1) The write succeeds. No error is returned. # # 2) An "out of memory" exception is thrown and: # # a) The statement has no effect, OR # b) The current transaction is rolled back, OR # c) The statement succeeds. This can only happen if the connection # is in auto-commit mode (after the statement is executed, so this # includes COMMIT statements). # # If the write operation eventually succeeds, zero is returned. If a # transaction is rolled back, non-zero is returned. # # Parameter $name is the name to use for the test case (or test cases). # The second parameter, $tbl, should be the name of the database table # being modified. Parameter $sql contains the SQL statement to test. # proc do_write_test {name tbl sql} { if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 } # Figure out an statement to get a checksum for table $tbl. db eval "SELECT * FROM $tbl" V break set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl" # Calculate the initial table checksum. set cksum1 [db one $cksumsql] if {$::DO_MALLOC_TEST } { set answers [list {1 {out of memory}} {0 {}}] set modes [list 100000 transient 1 persistent] } else { set answers [list {0 {}}] set modes [list 0 nofail] } set str [join $answers " OR "] foreach {nRepeat zName} $modes { for {set iFail 1} 1 {incr iFail} { if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat} set res [catchsql $sql] set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] if {$nFail==0} { do_test $name.$zName.$iFail [list set {} $res] {0 {}} return } else { if {[lsearch $answers $res]>=0} { set res $str } do_test $name.$zName.$iFail [list set {} $res] $str set cksum2 [db one $cksumsql] if {$cksum1 != $cksum2} return } } } } proc normal_list {l} { set ret [list] foreach elem $l {lappend ret $elem} set ret } do_write_test fts3_malloc-1.1 sqlite_master { CREATE VIRTUAL TABLE ft1 USING fts3(a, b) } do_write_test fts3_malloc-1.2 sqlite_master { CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]); } do_write_test fts3_malloc-1.3 sqlite_master { CREATE VIRTUAL TABLE ft3 USING fts3('a', "b"); } do_write_test fts3_malloc-1.4 sqlite_master { CREATE VIRTUAL TABLE ft4 USING fts3(`a`, 'fred''s column'); } do_error_test fts3_malloc-1.5 { CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown) } {unknown tokenizer: unknown} do_write_test fts3_malloc-1.6 sqlite_master { CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter) } # Test the xConnect/xDisconnect methods: #db eval { ATTACH 'test2.db' AS aux } #do_write_test fts3_malloc-1.6 aux.sqlite_master { # CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); #} #do_write_test fts3_malloc-1.6 aux.sqlite_master { # CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); #} do_test fts3_malloc-2.0 { execsql { DROP TABLE ft1; DROP TABLE ft2; DROP TABLE ft3; DROP TABLE ft4; DROP TABLE ft6; } execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) } for {set ii 1} {$ii < 32} {incr ii} { set a [list] set b [list] if {$ii & 0x01} {lappend a one ; lappend b neung} if {$ii & 0x02} {lappend a two ; lappend b song } if {$ii & 0x04} {lappend a three ; lappend b sahm } if {$ii & 0x08} {lappend a four ; lappend b see } if {$ii & 0x10} {lappend a five ; lappend b hah } execsql { INSERT INTO ft VALUES($a, $b) } } } {} foreach {tn sql result} { 1 "SELECT count(*) FROM sqlite_master" {5} 2 "SELECT * FROM ft WHERE docid = 1" {one neung} 3 "SELECT * FROM ft WHERE docid = 2" {two song} 4 "SELECT * FROM ft WHERE docid = 3" {{one two} {neung song}} 5 "SELECT a FROM ft" { {one} {two} {one two} {three} {one three} {two three} {one two three} {four} {one four} {two four} {one two four} {three four} {one three four} {two three four} {one two three four} {five} {one five} {two five} {one two five} {three five} {one three five} {two three five} {one two three five} {four five} {one four five} {two four five} {one two four five} {three four five} {one three four five} {two three four five} {one two three four five} } 6 "SELECT a FROM ft WHERE a MATCH 'one'" { {one} {one two} {one three} {one two three} {one four} {one two four} {one three four} {one two three four} {one five} {one two five} {one three five} {one two three five} {one four five} {one two four five} {one three four five} {one two three four five} } 7 "SELECT a FROM ft WHERE a MATCH 'o*'" { {one} {one two} {one three} {one two three} {one four} {one two four} {one three four} {one two three four} {one five} {one two five} {one three five} {one two three five} {one four five} {one two four five} {one three four five} {one two three four five} } 8 "SELECT a FROM ft WHERE a MATCH 'o* t*'" { {one two} {one three} {one two three} {one two four} {one three four} {one two three four} {one two five} {one three five} {one two three five} {one two four five} {one three four five} {one two three four five} } 9 "SELECT a FROM ft WHERE a MATCH '\"o* t*\"'" { {one two} {one three} {one two three} {one two four} {one three four} {one two three four} {one two five} {one three five} {one two three five} {one two four five} {one three four five} {one two three four five} } 10 {SELECT a FROM ft WHERE a MATCH '"o* f*"'} { {one four} {one five} {one four five} } 11 {SELECT a FROM ft WHERE a MATCH '"one two three"'} { {one two three} {one two three four} {one two three five} {one two three four five} } 12 {SELECT a FROM ft WHERE a MATCH '"two three four"'} { {two three four} {one two three four} {two three four five} {one two three four five} } 12 {SELECT a FROM ft WHERE a MATCH '"two three" five'} { {two three five} {one two three five} {two three four five} {one two three four five} } 13 {SELECT a FROM ft WHERE ft MATCH '"song sahm" hah'} { {two three five} {one two three five} {two three four five} {one two three four five} } 14 {SELECT a FROM ft WHERE b MATCH 'neung'} { {one} {one two} {one three} {one two three} {one four} {one two four} {one three four} {one two three four} {one five} {one two five} {one three five} {one two three five} {one four five} {one two four five} {one three four five} {one two three four five} } 15 {SELECT a FROM ft WHERE b MATCH '"neung song sahm"'} { {one two three} {one two three four} {one two three five} {one two three four five} } 16 {SELECT a FROM ft WHERE b MATCH 'hah "song sahm"'} { {two three five} {one two three five} {two three four five} {one two three four five} } 17 {SELECT a FROM ft WHERE b MATCH 'song OR sahm'} { {two} {one two} {three} {one three} {two three} {one two three} {two four} {one two four} {three four} {one three four} {two three four} {one two three four} {two five} {one two five} {three five} {one three five} {two three five} {one two three five} {two four five} {one two four five} {three four five} {one three four five} {two three four five} {one two three four five} } 18 {SELECT a FROM ft WHERE a MATCH 'three NOT two'} { {three} {one three} {three four} {one three four} {three five} {one three five} {three four five} {one three four five} } 19 {SELECT a FROM ft WHERE b MATCH 'sahm NOT song'} { {three} {one three} {three four} {one three four} {three five} {one three five} {three four five} {one three four five} } 20 {SELECT a FROM ft WHERE ft MATCH 'sahm NOT song'} { {three} {one three} {three four} {one three four} {three five} {one three five} {three four five} {one three four five} } 21 {SELECT a FROM ft WHERE b MATCH 'neung NEAR song NEAR sahm'} { {one two three} {one two three four} {one two three five} {one two three four five} } } { set result [normal_list $result] do_select_test fts3_malloc-2.$tn $sql $result } do_test fts3_malloc-3.0 { execsql BEGIN for {set ii 32} {$ii < 1024} {incr ii} { set a [list] set b [list] if {$ii & 0x0001} {lappend a one ; lappend b neung } if {$ii & 0x0002} {lappend a two ; lappend b song } if {$ii & 0x0004} {lappend a three ; lappend b sahm } if {$ii & 0x0008} {lappend a four ; lappend b see } if {$ii & 0x0010} {lappend a five ; lappend b hah } if {$ii & 0x0020} {lappend a six ; lappend b hok } if {$ii & 0x0040} {lappend a seven ; lappend b jet } if {$ii & 0x0080} {lappend a eight ; lappend b bairt } if {$ii & 0x0100} {lappend a nine ; lappend b gow } if {$ii & 0x0200} {lappend a ten ; lappend b sip } execsql { INSERT INTO ft VALUES($a, $b) } } execsql COMMIT } {} foreach {tn sql result} { 1 "SELECT count(*) FROM ft" {1023} 2 "SELECT a FROM ft WHERE a MATCH 'one two three four five six seven eight'" { {one two three four five six seven eight} {one two three four five six seven eight nine} {one two three four five six seven eight ten} {one two three four five six seven eight nine ten} } 3 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH 'o*'} { 512 262144 } 4 {SELECT count(*), sum(docid) FROM ft WHERE a MATCH '"two three four"'} { 128 66368 } } { set result [normal_list $result] do_select_test fts3_malloc-3.$tn $sql $result } do_test fts3_malloc-4.0 { execsql { DELETE FROM ft WHERE docid>=32 } } {} foreach {tn sql} { 1 "DELETE FROM ft WHERE ft MATCH 'one'" 2 "DELETE FROM ft WHERE ft MATCH 'three'" 3 "DELETE FROM ft WHERE ft MATCH 'five'" } { do_write_test fts3_malloc-4.1.$tn ft_content $sql } do_test fts3_malloc-4.2 { execsql { SELECT a FROM ft } } {two four {two four}} finish_test |