Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Merge the latest enhancements and fixes from trunk. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | apple-osx |
| Files: | files | file ages | folders |
| SHA1: |
d85774e0dcb6cb1958ecd85891ef785e |
| User & Date: | drh 2016-01-14 15:03:12 |
Context
|
2016-01-20
| ||
| 11:40 | Merge all recent enhancements from trunk. check-in: 3ed49691 user: drh tags: apple-osx | |
|
2016-01-14
| ||
| 15:03 | Merge the latest enhancements and fixes from trunk. check-in: d85774e0 user: drh tags: apple-osx | |
| 14:33 | Yet another change to FTS5 trying to get it to merge successfully into sessions. check-in: 8dedff3b user: drh tags: trunk | |
|
2016-01-11
| ||
| 13:03 | Import the version number change to 3.11.0. check-in: 132772d1 user: drh tags: apple-osx | |
Changes
Changes to Makefile.in.
175 175 func.lo global.lo hash.lo \ 176 176 icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ 177 177 main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ 178 178 memjournal.lo \ 179 179 mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ 180 180 notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ 181 181 pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ 182 - random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ 182 + random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ 183 183 table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ 184 184 update.lo util.lo vacuum.lo \ 185 185 vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ 186 186 vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ 187 187 utf.lo vtab.lo 188 188 189 189 # Object files for the amalgamation. ................................................................................ 1036 1036 fts5.c: $(FTS5_SRC) 1037 1037 $(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl 1038 1038 cp $(TOP)/ext/fts5/fts5.h . 1039 1039 1040 1040 fts5.lo: fts5.c $(HDR) $(EXTHDR) 1041 1041 $(LTCOMPILE) -DSQLITE_CORE -c fts5.c 1042 1042 1043 +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) 1044 + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c 1045 + 1043 1046 1044 1047 # Rules to build the 'testfixture' application. 1045 1048 # 1046 1049 # If using the amalgamation, use sqlite3.c directly to build the test 1047 1050 # fixture. Otherwise link against libsqlite3.la. (This distinction is 1048 1051 # necessary because the test fixture requires non-API symbols which are 1049 1052 # hidden when the library is built via the amalgamation). ................................................................................ 1138 1141 $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c 1139 1142 1140 1143 wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c 1141 1144 $(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.c $(TLIBS) 1142 1145 1143 1146 speedtest1$(TEXE): $(TOP)/test/speedtest1.c sqlite3.lo 1144 1147 $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) 1148 + 1149 +rbu$(EXE): $(TOP)/ext/rbu/rbu.c $(TOP)/ext/rbu/sqlite3rbu.c sqlite3.lo 1150 + $(LTLINK) -I. -o $@ $(TOP)/ext/rbu/rbu.c sqlite3.lo $(TLIBS) 1151 + 1152 +loadfts$(EXE): $(TOP)/tool/loadfts.c libsqlite3.la 1153 + $(LTLINK) $(TOP)/tool/loadfts.c libsqlite3.la -o $@ $(TLIBS) 1145 1154 1146 1155 # This target will fail if the SQLite amalgamation contains any exported 1147 1156 # symbols that do not begin with "sqlite3_". It is run as part of the 1148 1157 # releasetest.tcl script. 1149 1158 # 1150 1159 checksymbols: sqlite3.lo 1151 1160 nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0
Changes to Makefile.msc.
835 835 func.lo global.lo hash.lo \ 836 836 icu.lo insert.lo journal.lo legacy.lo loadext.lo \ 837 837 main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ 838 838 memjournal.lo \ 839 839 mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ 840 840 notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ 841 841 pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ 842 - random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ 842 + random.lo resolve.lo rowset.lo rtree.lo select.lo sqlite3rbu.lo status.lo \ 843 843 table.lo threads.lo tokenize.lo treeview.lo trigger.lo \ 844 844 update.lo util.lo vacuum.lo \ 845 845 vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ 846 846 vdbetrace.lo wal.lo walker.lo where.lo wherecode.lo whereexpr.lo \ 847 847 utf.lo vtab.lo 848 848 849 849 # Object files for the amalgamation. ................................................................................ 1710 1710 1711 1711 fts5_ext.lo: fts5.c $(HDR) $(EXTHDR) 1712 1712 $(LTCOMPILE) $(NO_WARN) -c fts5.c 1713 1713 1714 1714 fts5.dll: fts5_ext.lo 1715 1715 $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ fts5_ext.lo 1716 1716 1717 +sqlite3rbu.lo: $(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR) 1718 + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c 1719 + 1717 1720 # Rules to build the 'testfixture' application. 1718 1721 # 1719 1722 # If using the amalgamation, use sqlite3.c directly to build the test 1720 1723 # fixture. Otherwise link against libsqlite3.lib. (This distinction is 1721 1724 # necessary because the test fixture requires non-API symbols which are 1722 1725 # hidden when the library is built via the amalgamation). 1723 1726 # ................................................................................ 1831 1834 wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) 1832 1835 $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 1833 1836 $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1834 1837 1835 1838 speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) 1836 1839 $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ 1837 1840 $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) 1841 + 1842 +rbu.exe: $(TOP)\ext\rbu\rbu.c $(TOP)\ext\rbu\sqlite3rbu.c $(SQLITE3C) 1843 + $(LTLINK) $(NO_WARN) -I. -DSQLITE_ENABLE_RBU -Fe$@ $(TOP)\ext\rbu\rbu.c $(SQLITE3C) \ 1844 + $(LDFLAGS) $(LTLINKOPTS) 1838 1845 1839 1846 clean: 1840 1847 del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL 1841 1848 del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL 1842 1849 del /Q sqlite3.h opcodes.c opcodes.h 2>NUL 1843 1850 del /Q lemon.* lempar.c parse.* 2>NUL 1844 1851 del /Q mkkeywordhash.* keywordhash.h 2>NUL
Changes to autoconf/configure.ac.
69 69 # 70 70 AC_ARG_ENABLE(threadsafe, [AS_HELP_STRING( 71 71 [--enable-threadsafe], [build a thread-safe library [default=yes]])], 72 72 [], [enable_threadsafe=yes]) 73 73 THREADSAFE_FLAGS=-DSQLITE_THREADSAFE=0 74 74 if test x"$enable_threadsafe" != "xno"; then 75 75 THREADSAFE_FLAGS="-D_REENTRANT=1 -DSQLITE_THREADSAFE=1" 76 - AC_SEARCH_LIBS(pthread_create, pthread) 76 + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) 77 77 fi 78 78 AC_SUBST(THREADSAFE_FLAGS) 79 79 #----------------------------------------------------------------------- 80 80 81 81 #----------------------------------------------------------------------- 82 82 # --enable-dynamic-extensions 83 83 #
Changes to configure.
10460 10460 SQLITE_THREADSAFE=1 10461 10461 { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 10462 10462 $as_echo "yes" >&6; } 10463 10463 fi 10464 10464 10465 10465 10466 10466 if test "$SQLITE_THREADSAFE" = "1"; then 10467 - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 10468 -$as_echo_n "checking for library containing pthread_create... " >&6; } 10469 -if ${ac_cv_search_pthread_create+:} false; then : 10467 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_mutexattr_init" >&5 10468 +$as_echo_n "checking for library containing pthread_mutexattr_init... " >&6; } 10469 +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : 10470 10470 $as_echo_n "(cached) " >&6 10471 10471 else 10472 10472 ac_func_search_save_LIBS=$LIBS 10473 10473 cat confdefs.h - <<_ACEOF >conftest.$ac_ext 10474 10474 /* end confdefs.h. */ 10475 10475 10476 10476 /* Override any GCC internal prototype to avoid an error. 10477 10477 Use char because int might match the return type of a GCC 10478 10478 builtin and then its argument prototype would still apply. */ 10479 10479 #ifdef __cplusplus 10480 10480 extern "C" 10481 10481 #endif 10482 -char pthread_create (); 10482 +char pthread_mutexattr_init (); 10483 10483 int 10484 10484 main () 10485 10485 { 10486 -return pthread_create (); 10486 +return pthread_mutexattr_init (); 10487 10487 ; 10488 10488 return 0; 10489 10489 } 10490 10490 _ACEOF 10491 10491 for ac_lib in '' pthread; do 10492 10492 if test -z "$ac_lib"; then 10493 10493 ac_res="none required" 10494 10494 else 10495 10495 ac_res=-l$ac_lib 10496 10496 LIBS="-l$ac_lib $ac_func_search_save_LIBS" 10497 10497 fi 10498 10498 if ac_fn_c_try_link "$LINENO"; then : 10499 - ac_cv_search_pthread_create=$ac_res 10499 + ac_cv_search_pthread_mutexattr_init=$ac_res 10500 10500 fi 10501 10501 rm -f core conftest.err conftest.$ac_objext \ 10502 10502 conftest$ac_exeext 10503 - if ${ac_cv_search_pthread_create+:} false; then : 10503 + if ${ac_cv_search_pthread_mutexattr_init+:} false; then : 10504 10504 break 10505 10505 fi 10506 10506 done 10507 -if ${ac_cv_search_pthread_create+:} false; then : 10507 +if ${ac_cv_search_pthread_mutexattr_init+:} false; then : 10508 10508 10509 10509 else 10510 - ac_cv_search_pthread_create=no 10510 + ac_cv_search_pthread_mutexattr_init=no 10511 10511 fi 10512 10512 rm conftest.$ac_ext 10513 10513 LIBS=$ac_func_search_save_LIBS 10514 10514 fi 10515 -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 10516 -$as_echo "$ac_cv_search_pthread_create" >&6; } 10517 -ac_res=$ac_cv_search_pthread_create 10515 +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_mutexattr_init" >&5 10516 +$as_echo "$ac_cv_search_pthread_mutexattr_init" >&6; } 10517 +ac_res=$ac_cv_search_pthread_mutexattr_init 10518 10518 if test "$ac_res" != no; then : 10519 10519 test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" 10520 10520 10521 10521 fi 10522 10522 10523 10523 fi 10524 10524
Changes to configure.ac.
190 190 else 191 191 SQLITE_THREADSAFE=1 192 192 AC_MSG_RESULT([yes]) 193 193 fi 194 194 AC_SUBST(SQLITE_THREADSAFE) 195 195 196 196 if test "$SQLITE_THREADSAFE" = "1"; then 197 - AC_SEARCH_LIBS(pthread_create, pthread) 197 + AC_SEARCH_LIBS(pthread_mutexattr_init, pthread) 198 198 fi 199 199 200 200 ########## 201 201 # Do we want to support release 202 202 # 203 203 AC_ARG_ENABLE(releasemode, 204 204 AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
Changes to ext/fts5/fts5.h.
80 80 ** *pnToken to the number of tokens in column iCol of the current row. 81 81 ** 82 82 ** If parameter iCol is greater than or equal to the number of columns 83 83 ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. 84 84 ** an OOM condition or IO error), an appropriate SQLite error code is 85 85 ** returned. 86 86 ** 87 +** This function may be quite inefficient if used with an FTS5 table 88 +** created with the "columnsize=0" option. 89 +** 87 90 ** xColumnText: 88 91 ** This function attempts to retrieve the text of column iCol of the 89 92 ** current document. If successful, (*pz) is set to point to a buffer 90 93 ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes 91 94 ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, 92 95 ** if an error occurs, an SQLite error code is returned and the final values 93 96 ** of (*pz) and (*pn) are undefined. ................................................................................ 100 103 ** are numbered starting from zero. 101 104 ** 102 105 ** xInstCount: 103 106 ** Set *pnInst to the total number of occurrences of all phrases within 104 107 ** the query within the current row. Return SQLITE_OK if successful, or 105 108 ** an error code (i.e. SQLITE_NOMEM) if an error occurs. 106 109 ** 110 +** This API can be quite slow if used with an FTS5 table created with the 111 +** "detail=none" or "detail=column" option. If the FTS5 table is created 112 +** with either "detail=none" or "detail=column" and "content=" option 113 +** (i.e. if it is a contentless table), then this API always returns 0. 114 +** 107 115 ** xInst: 108 116 ** Query for the details of phrase match iIdx within the current row. 109 117 ** Phrase matches are numbered starting from zero, so the iIdx argument 110 118 ** should be greater than or equal to zero and smaller than the value 111 119 ** output by xInstCount(). 120 +** 121 +** Usually, output parameter *piPhrase is set to the phrase number, *piCol 122 +** to the column in which it occurs and *piOff the token offset of the 123 +** first token of the phrase. The exception is if the table was created 124 +** with the offsets=0 option specified. In this case *piOff is always 125 +** set to -1. 112 126 ** 113 127 ** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) 114 128 ** if an error occurs. 129 +** 130 +** This API can be quite slow if used with an FTS5 table created with the 131 +** "detail=none" or "detail=column" option. 115 132 ** 116 133 ** xRowid: 117 134 ** Returns the rowid of the current row. 118 135 ** 119 136 ** xTokenize: 120 137 ** Tokenize text using the tokenizer belonging to the FTS5 table. 121 138 ** ................................................................................ 192 209 ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient 193 210 ** to use, this API may be faster under some circumstances. To iterate 194 211 ** through instances of phrase iPhrase, use the following code: 195 212 ** 196 213 ** Fts5PhraseIter iter; 197 214 ** int iCol, iOff; 198 215 ** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); 199 -** iOff>=0; 216 +** iCol>=0; 200 217 ** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) 201 218 ** ){ 202 219 ** // An instance of phrase iPhrase at offset iOff of column iCol 203 220 ** } 204 221 ** 205 222 ** The Fts5PhraseIter structure is defined above. Applications should not 206 223 ** modify this structure directly - it should only be used as shown above 207 -** with the xPhraseFirst() and xPhraseNext() API methods. 224 +** with the xPhraseFirst() and xPhraseNext() API methods (and by 225 +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). 226 +** 227 +** This API can be quite slow if used with an FTS5 table created with the 228 +** "detail=none" or "detail=column" option. If the FTS5 table is created 229 +** with either "detail=none" or "detail=column" and "content=" option 230 +** (i.e. if it is a contentless table), then this API always iterates 231 +** through an empty set (all calls to xPhraseFirst() set iCol to -1). 208 232 ** 209 233 ** xPhraseNext() 210 234 ** See xPhraseFirst above. 235 +** 236 +** xPhraseFirstColumn() 237 +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() 238 +** and xPhraseNext() APIs described above. The difference is that instead 239 +** of iterating through all instances of a phrase in the current row, these 240 +** APIs are used to iterate through the set of columns in the current row 241 +** that contain one or more instances of a specified phrase. For example: 242 +** 243 +** Fts5PhraseIter iter; 244 +** int iCol; 245 +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); 246 +** iCol>=0; 247 +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) 248 +** ){ 249 +** // Column iCol contains at least one instance of phrase iPhrase 250 +** } 251 +** 252 +** This API can be quite slow if used with an FTS5 table created with the 253 +** "detail=none" option. If the FTS5 table is created with either 254 +** "detail=none" "content=" option (i.e. if it is a contentless table), 255 +** then this API always iterates through an empty set (all calls to 256 +** xPhraseFirstColumn() set iCol to -1). 257 +** 258 +** The information accessed using this API and its companion 259 +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext 260 +** (or xInst/xInstCount). The chief advantage of this API is that it is 261 +** significantly more efficient than those alternatives when used with 262 +** "detail=column" tables. 263 +** 264 +** xPhraseNextColumn() 265 +** See xPhraseFirstColumn above. 211 266 */ 212 267 struct Fts5ExtensionApi { 213 - int iVersion; /* Currently always set to 1 */ 268 + int iVersion; /* Currently always set to 3 */ 214 269 215 270 void *(*xUserData)(Fts5Context*); 216 271 217 272 int (*xColumnCount)(Fts5Context*); 218 273 int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); 219 274 int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); 220 275 ................................................................................ 236 291 237 292 int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, 238 293 int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) 239 294 ); 240 295 int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); 241 296 void *(*xGetAuxdata)(Fts5Context*, int bClear); 242 297 243 - void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); 298 + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); 244 299 void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); 300 + 301 + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); 302 + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); 245 303 }; 246 304 247 305 /* 248 306 ** CUSTOM AUXILIARY FUNCTIONS 249 307 *************************************************************************/ 250 308 251 309 /*************************************************************************
Changes to ext/fts5/fts5Int.h.
147 147 u8 *abUnindexed; /* True for unindexed columns */ 148 148 int nPrefix; /* Number of prefix indexes */ 149 149 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ 150 150 int eContent; /* An FTS5_CONTENT value */ 151 151 char *zContent; /* content table */ 152 152 char *zContentRowid; /* "content_rowid=" option value */ 153 153 int bColumnsize; /* "columnsize=" option value (dflt==1) */ 154 + int eDetail; /* FTS5_DETAIL_XXX value */ 154 155 char *zContentExprlist; 155 156 Fts5Tokenizer *pTok; 156 157 fts5_tokenizer *pTokApi; 157 158 158 159 /* Values loaded from the %_config table */ 159 160 int iCookie; /* Incremented when %_config is modified */ 160 161 int pgsz; /* Approximate page size used in %_data */ ................................................................................ 175 176 /* Current expected value of %_config table 'version' field */ 176 177 #define FTS5_CURRENT_VERSION 4 177 178 178 179 #define FTS5_CONTENT_NORMAL 0 179 180 #define FTS5_CONTENT_NONE 1 180 181 #define FTS5_CONTENT_EXTERNAL 2 181 182 183 +#define FTS5_DETAIL_FULL 0 184 +#define FTS5_DETAIL_NONE 1 185 +#define FTS5_DETAIL_COLUMNS 2 182 186 183 187 184 188 185 189 int sqlite3Fts5ConfigParse( 186 190 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** 187 191 ); 188 192 void sqlite3Fts5ConfigFree(Fts5Config*); ................................................................................ 288 292 /* Malloc utility */ 289 293 void *sqlite3Fts5MallocZero(int *pRc, int nByte); 290 294 char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); 291 295 292 296 /* Character set tests (like isspace(), isalpha() etc.) */ 293 297 int sqlite3Fts5IsBareword(char t); 294 298 299 + 300 +/* Bucket of terms object used by the integrity-check in offsets=0 mode. */ 301 +typedef struct Fts5Termset Fts5Termset; 302 +int sqlite3Fts5TermsetNew(Fts5Termset**); 303 +int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent); 304 +void sqlite3Fts5TermsetFree(Fts5Termset*); 305 + 295 306 /* 296 307 ** End of interface to code in fts5_buffer.c. 297 308 **************************************************************************/ 298 309 299 310 /************************************************************************** 300 311 ** Interface to code in fts5_index.c. fts5_index.c contains contains code 301 312 ** to access the data stored in the %_data table. ................................................................................ 323 334 ** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter); 324 335 ** 0==sqlite3Fts5IterEof(pIter); 325 336 ** sqlite3Fts5IterNext(pIter) 326 337 ** ){ 327 338 ** i64 iRowid = sqlite3Fts5IterRowid(pIter); 328 339 ** } 329 340 */ 341 + 342 +/* 343 +** Return a simple checksum value based on the arguments. 344 +*/ 345 +u64 sqlite3Fts5IndexEntryCksum( 346 + i64 iRowid, 347 + int iCol, 348 + int iPos, 349 + int iIdx, 350 + const char *pTerm, 351 + int nTerm 352 +); 353 + 354 +/* 355 +** Argument p points to a buffer containing utf-8 text that is n bytes in 356 +** size. Return the number of bytes in the nChar character prefix of the 357 +** buffer, or 0 if there are less than nChar characters in total. 358 +*/ 359 +int sqlite3Fts5IndexCharlenToBytelen( 360 + const char *p, 361 + int nByte, 362 + int nChar 363 +); 330 364 331 365 /* 332 366 ** Open a new iterator to iterate though all rowids that match the 333 367 ** specified token or token prefix. 334 368 */ 335 369 int sqlite3Fts5IndexQuery( 336 370 Fts5Index *p, /* FTS index to query */ ................................................................................ 409 443 */ 410 444 int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize); 411 445 int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); 412 446 413 447 /* 414 448 ** Functions called by the storage module as part of integrity-check. 415 449 */ 416 -u64 sqlite3Fts5IndexCksum(Fts5Config*,i64,int,int,const char*,int); 417 450 int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); 418 451 419 452 /* 420 453 ** Called during virtual module initialization to register UDF 421 454 ** fts5_decode() with SQLite 422 455 */ 423 456 int sqlite3Fts5IndexInit(sqlite3*); ................................................................................ 431 464 int sqlite3Fts5IndexReads(Fts5Index *p); 432 465 433 466 int sqlite3Fts5IndexReinit(Fts5Index *p); 434 467 int sqlite3Fts5IndexOptimize(Fts5Index *p); 435 468 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge); 436 469 437 470 int sqlite3Fts5IndexLoadConfig(Fts5Index *p); 471 + 472 +int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*); 438 473 439 474 /* 440 475 ** End of interface to code in fts5_index.c. 441 476 **************************************************************************/ 442 477 443 478 /************************************************************************** 444 479 ** Interface to code in fts5_varint.c. ................................................................................ 488 523 ** Interface to code in fts5_hash.c. 489 524 */ 490 525 typedef struct Fts5Hash Fts5Hash; 491 526 492 527 /* 493 528 ** Create a hash table, free a hash table. 494 529 */ 495 -int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize); 530 +int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize); 496 531 void sqlite3Fts5HashFree(Fts5Hash*); 497 532 498 533 int sqlite3Fts5HashWrite( 499 534 Fts5Hash*, 500 535 i64 iRowid, /* Rowid for this entry */ 501 536 int iCol, /* Column token appears in (-ve -> delete) */ 502 537 int iPos, /* Position of token within column */ ................................................................................ 624 659 625 660 /* Called during startup to register a UDF with SQLite */ 626 661 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); 627 662 628 663 int sqlite3Fts5ExprPhraseCount(Fts5Expr*); 629 664 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); 630 665 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); 666 + 667 +typedef struct Fts5PoslistPopulator Fts5PoslistPopulator; 668 +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int); 669 +int sqlite3Fts5ExprPopulatePoslists( 670 + Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int 671 +); 672 +void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); 673 +void sqlite3Fts5ExprClearEof(Fts5Expr*); 631 674 632 675 int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); 676 + 677 +int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); 633 678 634 679 /******************************************* 635 680 ** The fts5_expr.c API above this point is used by the other hand-written 636 681 ** C code in this module. The interfaces below this point are called by 637 682 ** the parser code in fts5parse.y. */ 638 683 639 684 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...);
Changes to ext/fts5/fts5_buffer.c.
286 286 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ 287 287 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ 288 288 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0 /* 0x70 .. 0x7F */ 289 289 }; 290 290 291 291 return (t & 0x80) || aBareword[(int)t]; 292 292 } 293 + 294 + 295 +/************************************************************************* 296 +*/ 297 +typedef struct Fts5TermsetEntry Fts5TermsetEntry; 298 +struct Fts5TermsetEntry { 299 + char *pTerm; 300 + int nTerm; 301 + int iIdx; /* Index (main or aPrefix[] entry) */ 302 + Fts5TermsetEntry *pNext; 303 +}; 304 + 305 +struct Fts5Termset { 306 + Fts5TermsetEntry *apHash[512]; 307 +}; 308 + 309 +int sqlite3Fts5TermsetNew(Fts5Termset **pp){ 310 + int rc = SQLITE_OK; 311 + *pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset)); 312 + return rc; 313 +} 314 + 315 +int sqlite3Fts5TermsetAdd( 316 + Fts5Termset *p, 317 + int iIdx, 318 + const char *pTerm, int nTerm, 319 + int *pbPresent 320 +){ 321 + int rc = SQLITE_OK; 322 + *pbPresent = 0; 323 + if( p ){ 324 + int i; 325 + int hash; 326 + Fts5TermsetEntry *pEntry; 327 + 328 + /* Calculate a hash value for this term */ 329 + hash = 104 + iIdx; 330 + for(i=0; i<nTerm; i++){ 331 + hash += (hash << 3) + (int)pTerm[i]; 332 + } 333 + hash = hash % ArraySize(p->apHash); 334 + 335 + for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ 336 + if( pEntry->iIdx==iIdx 337 + && pEntry->nTerm==nTerm 338 + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 339 + ){ 340 + *pbPresent = 1; 341 + break; 342 + } 343 + } 344 + 345 + if( pEntry==0 ){ 346 + pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm); 347 + if( pEntry ){ 348 + pEntry->pTerm = (char*)&pEntry[1]; 349 + pEntry->nTerm = nTerm; 350 + pEntry->iIdx = iIdx; 351 + memcpy(pEntry->pTerm, pTerm, nTerm); 352 + pEntry->pNext = p->apHash[hash]; 353 + p->apHash[hash] = pEntry; 354 + } 355 + } 356 + } 357 + 358 + return rc; 359 +} 360 + 361 +void sqlite3Fts5TermsetFree(Fts5Termset *p){ 362 + if( p ){ 363 + int i; 364 + for(i=0; i<ArraySize(p->apHash); i++){ 365 + Fts5TermsetEntry *pEntry = p->apHash[i]; 366 + while( pEntry ){ 367 + Fts5TermsetEntry *pDel = pEntry; 368 + pEntry = pEntry->pNext; 369 + sqlite3_free(pDel); 370 + } 371 + } 372 + sqlite3_free(p); 373 + } 374 +} 375 + 293 376 294 377
Changes to ext/fts5/fts5_config.c.
10 10 ** 11 11 ****************************************************************************** 12 12 ** 13 13 ** This is an SQLite module implementing full-text search. 14 14 */ 15 15 16 16 17 - 18 17 #include "fts5Int.h" 19 18 20 19 #define FTS5_DEFAULT_PAGE_SIZE 4050 21 20 #define FTS5_DEFAULT_AUTOMERGE 4 22 21 #define FTS5_DEFAULT_CRISISMERGE 16 23 22 #define FTS5_DEFAULT_HASHSIZE (1024*1024) 24 23 ................................................................................ 191 190 assert( 0==fts5_iswhitespace(z[0]) ); 192 191 quote = z[0]; 193 192 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ 194 193 fts5Dequote(z); 195 194 } 196 195 } 197 196 197 + 198 +struct Fts5Enum { 199 + const char *zName; 200 + int eVal; 201 +}; 202 +typedef struct Fts5Enum Fts5Enum; 203 + 204 +static int fts5ConfigSetEnum( 205 + const Fts5Enum *aEnum, 206 + const char *zEnum, 207 + int *peVal 208 +){ 209 + int nEnum = strlen(zEnum); 210 + int i; 211 + int iVal = -1; 212 + 213 + for(i=0; aEnum[i].zName; i++){ 214 + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ 215 + if( iVal>=0 ) return SQLITE_ERROR; 216 + iVal = aEnum[i].eVal; 217 + } 218 + } 219 + 220 + *peVal = iVal; 221 + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; 222 +} 223 + 198 224 /* 199 225 ** Parse a "special" CREATE VIRTUAL TABLE directive and update 200 226 ** configuration object pConfig as appropriate. 201 227 ** 202 228 ** If successful, object pConfig is updated and SQLITE_OK returned. If 203 229 ** an error occurs, an SQLite error code is returned and an error message 204 230 ** may be left in *pzErr. It is the responsibility of the caller to ................................................................................ 340 366 *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); 341 367 rc = SQLITE_ERROR; 342 368 }else{ 343 369 pConfig->bColumnsize = (zArg[0]=='1'); 344 370 } 345 371 return rc; 346 372 } 373 + 374 + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ 375 + const Fts5Enum aDetail[] = { 376 + { "none", FTS5_DETAIL_NONE }, 377 + { "full", FTS5_DETAIL_FULL }, 378 + { "columns", FTS5_DETAIL_COLUMNS }, 379 + { 0, 0 } 380 + }; 381 + 382 + if( (rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail)) ){ 383 + *pzErr = sqlite3_mprintf("malformed detail=... directive"); 384 + } 385 + return rc; 386 + } 347 387 348 388 *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); 349 389 return SQLITE_ERROR; 350 390 } 351 391 352 392 /* 353 393 ** Allocate an instance of the default tokenizer ("simple") at ................................................................................ 496 536 497 537 nByte = nArg * (sizeof(char*) + sizeof(u8)); 498 538 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); 499 539 pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; 500 540 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); 501 541 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); 502 542 pRet->bColumnsize = 1; 543 + pRet->eDetail = FTS5_DETAIL_FULL; 503 544 #ifdef SQLITE_DEBUG 504 545 pRet->bPrefixIndex = 1; 505 546 #endif 506 547 if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ 507 548 *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); 508 549 rc = SQLITE_ERROR; 509 550 }
Changes to ext/fts5/fts5_expr.c.
36 36 #include <stdio.h> 37 37 void sqlite3Fts5ParserTrace(FILE*, char*); 38 38 #endif 39 39 40 40 41 41 struct Fts5Expr { 42 42 Fts5Index *pIndex; 43 + Fts5Config *pConfig; 43 44 Fts5ExprNode *pRoot; 44 45 int bDesc; /* Iterate in descending rowid order */ 45 46 int nPhrase; /* Number of phrases in expression */ 46 47 Fts5ExprPhrase **apExprPhrase; /* Pointers to phrase objects */ 47 48 }; 48 49 49 50 /* ................................................................................ 231 232 *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); 232 233 if( pNew==0 ){ 233 234 sParse.rc = SQLITE_NOMEM; 234 235 sqlite3Fts5ParseNodeFree(sParse.pExpr); 235 236 }else{ 236 237 pNew->pRoot = sParse.pExpr; 237 238 pNew->pIndex = 0; 239 + pNew->pConfig = pConfig; 238 240 pNew->apExprPhrase = sParse.apPhrase; 239 241 pNew->nPhrase = sParse.nPhrase; 240 242 sParse.apPhrase = 0; 241 243 } 242 244 } 243 245 244 246 sqlite3_free(sParse.apPhrase); ................................................................................ 295 297 if( pbEof && bRetValid==0 ) *pbEof = 1; 296 298 return iRet; 297 299 } 298 300 299 301 /* 300 302 ** Argument pTerm must be a synonym iterator. 301 303 */ 302 -static int fts5ExprSynonymPoslist( 304 +static int fts5ExprSynonymList( 303 305 Fts5ExprTerm *pTerm, 306 + int bCollist, 304 307 Fts5Colset *pColset, 305 308 i64 iRowid, 306 309 int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ 307 310 u8 **pa, int *pn 308 311 ){ 309 312 Fts5PoslistReader aStatic[4]; 310 313 Fts5PoslistReader *aIter = aStatic; ................................................................................ 315 318 316 319 assert( pTerm->pSynonym ); 317 320 for(p=pTerm; p; p=p->pSynonym){ 318 321 Fts5IndexIter *pIter = p->pIter; 319 322 if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ 320 323 const u8 *a; 321 324 int n; 322 - i64 dummy; 323 - rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); 325 + 326 + if( bCollist ){ 327 + rc = sqlite3Fts5IterCollist(pIter, &a, &n); 328 + }else{ 329 + i64 dummy; 330 + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); 331 + } 332 + 324 333 if( rc!=SQLITE_OK ) goto synonym_poslist_out; 334 + if( n==0 ) continue; 325 335 if( nIter==nAlloc ){ 326 336 int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; 327 337 Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); 328 338 if( aNew==0 ){ 329 339 rc = SQLITE_NOMEM; 330 340 goto synonym_poslist_out; 331 341 } ................................................................................ 418 428 for(i=0; i<pPhrase->nTerm; i++){ 419 429 Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; 420 430 i64 dummy; 421 431 int n = 0; 422 432 int bFlag = 0; 423 433 const u8 *a = 0; 424 434 if( pTerm->pSynonym ){ 425 - rc = fts5ExprSynonymPoslist( 426 - pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n 435 + rc = fts5ExprSynonymList( 436 + pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n 427 437 ); 428 438 }else{ 429 439 rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); 430 440 } 431 441 if( rc!=SQLITE_OK ) goto ismatch_out; 432 442 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); 433 443 aIter[i].bFlag = (u8)bFlag; ................................................................................ 753 763 static int fts5ExprNearTest( 754 764 int *pRc, 755 765 Fts5Expr *pExpr, /* Expression that pNear is a part of */ 756 766 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ 757 767 ){ 758 768 Fts5ExprNearset *pNear = pNode->pNear; 759 769 int rc = *pRc; 760 - int i; 761 - 762 - /* Check that each phrase in the nearset matches the current row. 763 - ** Populate the pPhrase->poslist buffers at the same time. If any 764 - ** phrase is not a match, break out of the loop early. */ 765 - for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ 766 - Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 767 - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ 768 - int bMatch = 0; 769 - rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); 770 - if( bMatch==0 ) break; 771 - }else{ 772 - rc = sqlite3Fts5IterPoslistBuffer( 773 - pPhrase->aTerm[0].pIter, &pPhrase->poslist 774 - ); 775 - } 776 - } 777 - 778 - *pRc = rc; 779 - if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ 780 - return 1; 781 - } 782 - 783 - return 0; 770 + 771 + if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){ 772 + Fts5ExprTerm *pTerm; 773 + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; 774 + pPhrase->poslist.n = 0; 775 + for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ 776 + Fts5IndexIter *pIter = pTerm->pIter; 777 + if( sqlite3Fts5IterEof(pIter)==0 ){ 778 + int n; 779 + i64 iRowid; 780 + rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid); 781 + if( rc!=SQLITE_OK ){ 782 + *pRc = rc; 783 + return 0; 784 + }else if( iRowid==pNode->iRowid && n>0 ){ 785 + pPhrase->poslist.n = 1; 786 + } 787 + } 788 + } 789 + return pPhrase->poslist.n; 790 + }else{ 791 + int i; 792 + 793 + /* Check that each phrase in the nearset matches the current row. 794 + ** Populate the pPhrase->poslist buffers at the same time. If any 795 + ** phrase is not a match, break out of the loop early. */ 796 + for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){ 797 + Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 798 + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ 799 + int bMatch = 0; 800 + rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch); 801 + if( bMatch==0 ) break; 802 + }else{ 803 + rc = sqlite3Fts5IterPoslistBuffer( 804 + pPhrase->aTerm[0].pIter, &pPhrase->poslist 805 + ); 806 + } 807 + } 808 + 809 + *pRc = rc; 810 + if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){ 811 + return 1; 812 + } 813 + return 0; 814 + } 784 815 } 785 816 786 817 static int fts5ExprTokenTest( 787 818 Fts5Expr *pExpr, /* Expression that pNear is a part of */ 788 819 Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ 789 820 ){ 790 821 /* As this "NEAR" object is actually a single phrase that consists ................................................................................ 1214 1245 } 1215 1246 assert( rc!=SQLITE_OK || cmp<=0 ); 1216 1247 if( cmp || p2->bNomatch ) break; 1217 1248 rc = fts5ExprNodeNext(pExpr, p1, 0, 0); 1218 1249 } 1219 1250 pNode->bEof = p1->bEof; 1220 1251 pNode->iRowid = p1->iRowid; 1252 + if( p1->bEof ){ 1253 + fts5ExprNodeZeroPoslist(p2); 1254 + } 1221 1255 break; 1222 1256 } 1223 1257 } 1224 1258 } 1225 1259 return rc; 1226 1260 } 1227 1261 ................................................................................ 1599 1633 sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; 1600 1634 } 1601 1635 } 1602 1636 1603 1637 if( rc==SQLITE_OK ){ 1604 1638 /* All the allocations succeeded. Put the expression object together. */ 1605 1639 pNew->pIndex = pExpr->pIndex; 1640 + pNew->pConfig = pExpr->pConfig; 1606 1641 pNew->nPhrase = 1; 1607 1642 pNew->apExprPhrase[0] = sCtx.pPhrase; 1608 1643 pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase; 1609 1644 pNew->pRoot->pNear->nPhrase = 1; 1610 1645 sCtx.pPhrase->pNode = pNew->pRoot; 1611 1646 1612 1647 if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ ................................................................................ 1740 1775 } 1741 1776 1742 1777 void sqlite3Fts5ParseSetColset( 1743 1778 Fts5Parse *pParse, 1744 1779 Fts5ExprNearset *pNear, 1745 1780 Fts5Colset *pColset 1746 1781 ){ 1782 + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ 1783 + pParse->rc = SQLITE_ERROR; 1784 + pParse->zErr = sqlite3_mprintf( 1785 + "fts5: column queries are not supported (detail=none)" 1786 + ); 1787 + sqlite3_free(pColset); 1788 + return; 1789 + } 1790 + 1747 1791 if( pNear ){ 1748 1792 pNear->pColset = pColset; 1749 1793 }else{ 1750 1794 sqlite3_free(pColset); 1751 1795 } 1752 1796 } 1753 1797 ................................................................................ 1801 1845 pRet->eType = eType; 1802 1846 pRet->pNear = pNear; 1803 1847 if( eType==FTS5_STRING ){ 1804 1848 int iPhrase; 1805 1849 for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ 1806 1850 pNear->apPhrase[iPhrase]->pNode = pRet; 1807 1851 } 1808 - if( pNear->nPhrase==1 1809 - && pNear->apPhrase[0]->nTerm==1 1810 - && pNear->apPhrase[0]->aTerm[0].pSynonym==0 1811 - ){ 1812 - pRet->eType = FTS5_TERM; 1852 + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ 1853 + if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ 1854 + pRet->eType = FTS5_TERM; 1855 + } 1856 + }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ 1857 + assert( pParse->rc==SQLITE_OK ); 1858 + pParse->rc = SQLITE_ERROR; 1859 + assert( pParse->zErr==0 ); 1860 + pParse->zErr = sqlite3_mprintf( 1861 + "fts5: %s queries are not supported (detail!=full)", 1862 + pNear->nPhrase==1 ? "phrase": "NEAR" 1863 + ); 1864 + sqlite3_free(pRet); 1865 + pRet = 0; 1813 1866 } 1814 1867 }else{ 1815 1868 fts5ExprAddChildren(pRet, pLeft); 1816 1869 fts5ExprAddChildren(pRet, pRight); 1817 1870 } 1818 1871 } 1819 1872 } ................................................................................ 1919 1972 for(i=0; i<pNear->nPhrase; i++){ 1920 1973 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 1921 1974 1922 1975 zRet = fts5PrintfAppend(zRet, " {"); 1923 1976 for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ 1924 1977 char *zTerm = pPhrase->aTerm[iTerm].zTerm; 1925 1978 zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); 1979 + if( pPhrase->aTerm[iTerm].bPrefix ){ 1980 + zRet = fts5PrintfAppend(zRet, "*"); 1981 + } 1926 1982 } 1927 1983 1928 1984 if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); 1929 1985 if( zRet==0 ) return 0; 1930 1986 } 1931 1987 1932 1988 }else{ ................................................................................ 2231 2287 nRet = pPhrase->poslist.n; 2232 2288 }else{ 2233 2289 *pa = 0; 2234 2290 nRet = 0; 2235 2291 } 2236 2292 return nRet; 2237 2293 } 2294 + 2295 +struct Fts5PoslistPopulator { 2296 + Fts5PoslistWriter writer; 2297 + int bOk; /* True if ok to populate */ 2298 + int bMiss; 2299 +}; 2300 + 2301 +Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ 2302 + Fts5PoslistPopulator *pRet; 2303 + pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); 2304 + if( pRet ){ 2305 + int i; 2306 + memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); 2307 + for(i=0; i<pExpr->nPhrase; i++){ 2308 + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; 2309 + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; 2310 + assert( pExpr->apExprPhrase[i]->nTerm==1 ); 2311 + if( bLive && 2312 + (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) 2313 + ){ 2314 + pRet[i].bMiss = 1; 2315 + }else{ 2316 + pBuf->n = 0; 2317 + } 2318 + } 2319 + } 2320 + return pRet; 2321 +} 2322 + 2323 +struct Fts5ExprCtx { 2324 + Fts5Expr *pExpr; 2325 + Fts5PoslistPopulator *aPopulator; 2326 + i64 iOff; 2327 +}; 2328 +typedef struct Fts5ExprCtx Fts5ExprCtx; 2329 + 2330 +/* 2331 +** TODO: Make this more efficient! 2332 +*/ 2333 +static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ 2334 + int i; 2335 + for(i=0; i<pColset->nCol; i++){ 2336 + if( pColset->aiCol[i]==iCol ) return 1; 2337 + } 2338 + return 0; 2339 +} 2340 + 2341 +static int fts5ExprPopulatePoslistsCb( 2342 + void *pCtx, /* Copy of 2nd argument to xTokenize() */ 2343 + int tflags, /* Mask of FTS5_TOKEN_* flags */ 2344 + const char *pToken, /* Pointer to buffer containing token */ 2345 + int nToken, /* Size of token in bytes */ 2346 + int iStart, /* Byte offset of token within input text */ 2347 + int iEnd /* Byte offset of end of token within input text */ 2348 +){ 2349 + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; 2350 + Fts5Expr *pExpr = p->pExpr; 2351 + int i; 2352 + 2353 + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; 2354 + for(i=0; i<pExpr->nPhrase; i++){ 2355 + Fts5ExprTerm *pTerm; 2356 + if( p->aPopulator[i].bOk==0 ) continue; 2357 + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ 2358 + int nTerm = strlen(pTerm->zTerm); 2359 + if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) 2360 + && memcmp(pTerm->zTerm, pToken, nTerm)==0 2361 + ){ 2362 + int rc = sqlite3Fts5PoslistWriterAppend( 2363 + &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff 2364 + ); 2365 + if( rc ) return rc; 2366 + break; 2367 + } 2368 + } 2369 + } 2370 + return SQLITE_OK; 2371 +} 2372 + 2373 +int sqlite3Fts5ExprPopulatePoslists( 2374 + Fts5Config *pConfig, 2375 + Fts5Expr *pExpr, 2376 + Fts5PoslistPopulator *aPopulator, 2377 + int iCol, 2378 + const char *z, int n 2379 +){ 2380 + int i; 2381 + Fts5ExprCtx sCtx; 2382 + sCtx.pExpr = pExpr; 2383 + sCtx.aPopulator = aPopulator; 2384 + sCtx.iOff = (((i64)iCol) << 32) - 1; 2385 + 2386 + for(i=0; i<pExpr->nPhrase; i++){ 2387 + Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; 2388 + Fts5Colset *pColset = pNode->pNear->pColset; 2389 + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) 2390 + || aPopulator[i].bMiss 2391 + ){ 2392 + aPopulator[i].bOk = 0; 2393 + }else{ 2394 + aPopulator[i].bOk = 1; 2395 + } 2396 + } 2397 + 2398 + return sqlite3Fts5Tokenize(pConfig, 2399 + FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb 2400 + ); 2401 +} 2402 + 2403 +static void fts5ExprClearPoslists(Fts5ExprNode *pNode){ 2404 + if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){ 2405 + pNode->pNear->apPhrase[0]->poslist.n = 0; 2406 + }else{ 2407 + int i; 2408 + for(i=0; i<pNode->nChild; i++){ 2409 + fts5ExprClearPoslists(pNode->apChild[i]); 2410 + } 2411 + } 2412 +} 2413 + 2414 +static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ 2415 + if( pNode ){ 2416 + pNode->iRowid = iRowid; 2417 + pNode->bEof = 0; 2418 + switch( pNode->eType ){ 2419 + case FTS5_TERM: 2420 + case FTS5_STRING: 2421 + return (pNode->pNear->apPhrase[0]->poslist.n>0); 2422 + 2423 + case FTS5_AND: { 2424 + int i; 2425 + for(i=0; i<pNode->nChild; i++){ 2426 + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){ 2427 + fts5ExprClearPoslists(pNode); 2428 + return 0; 2429 + } 2430 + } 2431 + break; 2432 + } 2433 + 2434 + case FTS5_OR: { 2435 + int i; 2436 + int bRet = 0; 2437 + for(i=0; i<pNode->nChild; i++){ 2438 + if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){ 2439 + bRet = 1; 2440 + } 2441 + } 2442 + if( bRet==0 ){ 2443 + fts5ExprClearPoslists(pNode); 2444 + } 2445 + return bRet; 2446 + } 2447 + 2448 + default: { 2449 + assert( pNode->eType==FTS5_NOT ); 2450 + if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid) 2451 + || 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid) 2452 + ){ 2453 + fts5ExprClearPoslists(pNode); 2454 + return 0; 2455 + } 2456 + break; 2457 + } 2458 + } 2459 + } 2460 + return 1; 2461 +} 2462 + 2463 +void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ 2464 + fts5ExprCheckPoslists(pExpr->pRoot, iRowid); 2465 +} 2466 + 2467 +static void fts5ExprClearEof(Fts5ExprNode *pNode){ 2468 + int i; 2469 + for(i=0; i<pNode->nChild; i++){ 2470 + fts5ExprClearEof(pNode->apChild[i]); 2471 + } 2472 + pNode->bEof = 0; 2473 +} 2474 +void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ 2475 + fts5ExprClearEof(pExpr->pRoot); 2476 +} 2477 + 2478 +/* 2479 +** This function is only called for detail=columns tables. 2480 +*/ 2481 +int sqlite3Fts5ExprPhraseCollist( 2482 + Fts5Expr *pExpr, 2483 + int iPhrase, 2484 + const u8 **ppCollist, 2485 + int *pnCollist 2486 +){ 2487 + Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; 2488 + Fts5ExprNode *pNode = pPhrase->pNode; 2489 + int rc = SQLITE_OK; 2490 + 2491 + assert( iPhrase>=0 && iPhrase<pExpr->nPhrase ); 2492 + if( pNode->bEof==0 2493 + && pNode->iRowid==pExpr->pRoot->iRowid 2494 + && pPhrase->poslist.n>0 2495 + ){ 2496 + Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; 2497 + if( pTerm->pSynonym ){ 2498 + int bDel = 0; 2499 + u8 *a; 2500 + rc = fts5ExprSynonymList( 2501 + pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist 2502 + ); 2503 + if( bDel ){ 2504 + sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a); 2505 + *ppCollist = pPhrase->poslist.p; 2506 + sqlite3_free(a); 2507 + }else{ 2508 + *ppCollist = a; 2509 + } 2510 + }else{ 2511 + sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist); 2512 + } 2513 + }else{ 2514 + *ppCollist = 0; 2515 + *pnCollist = 0; 2516 + } 2517 + 2518 + return rc; 2519 +} 2520 +
Changes to ext/fts5/fts5_hash.c.
22 22 ** This file contains the implementation of an in-memory hash table used 23 23 ** to accumuluate "term -> doclist" content before it is flused to a level-0 24 24 ** segment. 25 25 */ 26 26 27 27 28 28 struct Fts5Hash { 29 + int eDetail; /* Copy of Fts5Config.eDetail */ 29 30 int *pnByte; /* Pointer to bytes counter */ 30 31 int nEntry; /* Number of entries currently in hash */ 31 32 int nSlot; /* Size of aSlot[] array */ 32 33 Fts5HashEntry *pScan; /* Current ordered scan item */ 33 34 Fts5HashEntry **aSlot; /* Array of hash slots */ 34 35 }; 35 36 ................................................................................ 58 59 Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ 59 60 Fts5HashEntry *pScanNext; /* Next entry in sorted order */ 60 61 61 62 int nAlloc; /* Total size of allocation */ 62 63 int iSzPoslist; /* Offset of space for 4-byte poslist size */ 63 64 int nData; /* Total bytes of data (incl. structure) */ 64 65 u8 bDel; /* Set delete-flag @ iSzPoslist */ 66 + u8 bContent; /* Set content-flag (detail=none mode) */ 65 67 66 68 int iCol; /* Column of last value written */ 67 69 int iPos; /* Position of last value written */ 68 70 i64 iRowid; /* Rowid of last value written */ 69 71 char zKey[8]; /* Nul-terminated entry key */ 70 72 }; 71 73 ................................................................................ 75 77 #define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) 76 78 77 79 78 80 79 81 /* 80 82 ** Allocate a new hash table. 81 83 */ 82 -int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){ 84 +int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){ 83 85 int rc = SQLITE_OK; 84 86 Fts5Hash *pNew; 85 87 86 88 *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); 87 89 if( pNew==0 ){ 88 90 rc = SQLITE_NOMEM; 89 91 }else{ 90 92 int nByte; 91 93 memset(pNew, 0, sizeof(Fts5Hash)); 92 94 pNew->pnByte = pnByte; 95 + pNew->eDetail = pConfig->eDetail; 93 96 94 97 pNew->nSlot = 1024; 95 98 nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; 96 99 pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); 97 100 if( pNew->aSlot==0 ){ 98 101 sqlite3_free(pNew); 99 102 *ppNew = 0; ................................................................................ 178 181 179 182 sqlite3_free(apOld); 180 183 pHash->nSlot = nNew; 181 184 pHash->aSlot = apNew; 182 185 return SQLITE_OK; 183 186 } 184 187 185 -static void fts5HashAddPoslistSize(Fts5HashEntry *p){ 188 +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ 186 189 if( p->iSzPoslist ){ 187 190 u8 *pPtr = (u8*)p; 188 - int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ 189 - int nPos = nSz*2 + p->bDel; /* Value of nPos field */ 190 - 191 - assert( p->bDel==0 || p->bDel==1 ); 192 - if( nPos<=127 ){ 193 - pPtr[p->iSzPoslist] = (u8)nPos; 191 + if( pHash->eDetail==FTS5_DETAIL_NONE ){ 192 + assert( p->nData==p->iSzPoslist ); 193 + if( p->bDel ){ 194 + pPtr[p->nData++] = 0x00; 195 + if( p->bContent ){ 196 + pPtr[p->nData++] = 0x00; 197 + } 198 + } 194 199 }else{ 195 - int nByte = sqlite3Fts5GetVarintLen((u32)nPos); 196 - memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); 197 - sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); 198 - p->nData += (nByte-1); 200 + int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ 201 + int nPos = nSz*2 + p->bDel; /* Value of nPos field */ 202 + 203 + assert( p->bDel==0 || p->bDel==1 ); 204 + if( nPos<=127 ){ 205 + pPtr[p->iSzPoslist] = (u8)nPos; 206 + }else{ 207 + int nByte = sqlite3Fts5GetVarintLen((u32)nPos); 208 + memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); 209 + sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); 210 + p->nData += (nByte-1); 211 + } 199 212 } 200 - p->bDel = 0; 213 + 201 214 p->iSzPoslist = 0; 215 + p->bDel = 0; 216 + p->bContent = 0; 202 217 } 203 218 } 204 219 220 +/* 221 +** Add an entry to the in-memory hash table. The key is the concatenation 222 +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos). 223 +** 224 +** (bByte || pToken) -> (iRowid,iCol,iPos) 225 +** 226 +** Or, if iCol is negative, then the value is a delete marker. 227 +*/ 205 228 int sqlite3Fts5HashWrite( 206 229 Fts5Hash *pHash, 207 230 i64 iRowid, /* Rowid for this entry */ 208 231 int iCol, /* Column token appears in (-ve -> delete) */ 209 232 int iPos, /* Position of token within column */ 210 233 char bByte, /* First byte of token */ 211 234 const char *pToken, int nToken /* Token to add or remove to or from index */ 212 235 ){ 213 236 unsigned int iHash; 214 237 Fts5HashEntry *p; 215 238 u8 *pPtr; 216 239 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ 240 + int bNew; /* If non-delete entry should be written */ 241 + 242 + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); 217 243 218 244 /* Attempt to locate an existing hash entry */ 219 245 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); 220 246 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ 221 247 if( p->zKey[0]==bByte 222 248 && memcmp(&p->zKey[1], pToken, nToken)==0 223 249 && p->zKey[nToken+1]==0 ................................................................................ 224 250 ){ 225 251 break; 226 252 } 227 253 } 228 254 229 255 /* If an existing hash entry cannot be found, create a new one. */ 230 256 if( p==0 ){ 257 + /* Figure out how much space to allocate */ 231 258 int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; 232 259 if( nByte<128 ) nByte = 128; 233 260 261 + /* Grow the Fts5Hash.aSlot[] array if necessary. */ 234 262 if( (pHash->nEntry*2)>=pHash->nSlot ){ 235 263 int rc = fts5HashResize(pHash); 236 264 if( rc!=SQLITE_OK ) return rc; 237 265 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); 238 266 } 239 267 268 + /* Allocate new Fts5HashEntry and add it to the hash table. */ 240 269 p = (Fts5HashEntry*)sqlite3_malloc(nByte); 241 270 if( !p ) return SQLITE_NOMEM; 242 271 memset(p, 0, FTS5_HASHENTRYSIZE); 243 272 p->nAlloc = nByte; 244 273 p->zKey[0] = bByte; 245 274 memcpy(&p->zKey[1], pToken, nToken); 246 275 assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); 247 276 p->zKey[nToken+1] = '\0'; 248 277 p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; 249 - p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); 250 - p->iSzPoslist = p->nData; 251 - p->nData += 1; 252 - p->iRowid = iRowid; 253 278 p->pHashNext = pHash->aSlot[iHash]; 254 279 pHash->aSlot[iHash] = p; 255 280 pHash->nEntry++; 281 + 282 + /* Add the first rowid field to the hash-entry */ 283 + p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); 284 + p->iRowid = iRowid; 285 + 286 + p->iSzPoslist = p->nData; 287 + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ 288 + p->nData += 1; 289 + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); 290 + } 291 + 256 292 nIncr += p->nData; 257 - } 258 - 259 - /* Check there is enough space to append a new entry. Worst case scenario 260 - ** is: 261 - ** 262 - ** + 9 bytes for a new rowid, 263 - ** + 4 byte reserved for the "poslist size" varint. 264 - ** + 1 byte for a "new column" byte, 265 - ** + 3 bytes for a new column number (16-bit max) as a varint, 266 - ** + 5 bytes for the new position offset (32-bit max). 267 - */ 268 - if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ 269 - int nNew = p->nAlloc * 2; 270 - Fts5HashEntry *pNew; 271 - Fts5HashEntry **pp; 272 - pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); 273 - if( pNew==0 ) return SQLITE_NOMEM; 274 - pNew->nAlloc = nNew; 275 - for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); 276 - *pp = pNew; 277 - p = pNew; 293 + }else{ 294 + 295 + /* Appending to an existing hash-entry. Check that there is enough 296 + ** space to append the largest possible new entry. Worst case scenario 297 + ** is: 298 + ** 299 + ** + 9 bytes for a new rowid, 300 + ** + 4 byte reserved for the "poslist size" varint. 301 + ** + 1 byte for a "new column" byte, 302 + ** + 3 bytes for a new column number (16-bit max) as a varint, 303 + ** + 5 bytes for the new position offset (32-bit max). 304 + */ 305 + if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ 306 + int nNew = p->nAlloc * 2; 307 + Fts5HashEntry *pNew; 308 + Fts5HashEntry **pp; 309 + pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); 310 + if( pNew==0 ) return SQLITE_NOMEM; 311 + pNew->nAlloc = nNew; 312 + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); 313 + *pp = pNew; 314 + p = pNew; 315 + } 316 + nIncr -= p->nData; 278 317 } 318 + assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) ); 319 + 279 320 pPtr = (u8*)p; 280 - nIncr -= p->nData; 281 321 282 322 /* If this is a new rowid, append the 4-byte size field for the previous 283 323 ** entry, and the new rowid for this entry. */ 284 324 if( iRowid!=p->iRowid ){ 285 - fts5HashAddPoslistSize(p); 325 + fts5HashAddPoslistSize(pHash, p); 286 326 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); 327 + p->iRowid = iRowid; 328 + bNew = 1; 287 329 p->iSzPoslist = p->nData; 288 - p->nData += 1; 289 - p->iCol = 0; 290 - p->iPos = 0; 291 - p->iRowid = iRowid; 330 + if( pHash->eDetail!=FTS5_DETAIL_NONE ){ 331 + p->nData += 1; 332 + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); 333 + p->iPos = 0; 334 + } 292 335 } 293 336 294 337 if( iCol>=0 ){ 295 - /* Append a new column value, if necessary */ 296 - assert( iCol>=p->iCol ); 297 - if( iCol!=p->iCol ){ 298 - pPtr[p->nData++] = 0x01; 299 - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); 300 - p->iCol = iCol; 301 - p->iPos = 0; 338 + if( pHash->eDetail==FTS5_DETAIL_NONE ){ 339 + p->bContent = 1; 340 + }else{ 341 + /* Append a new column value, if necessary */ 342 + assert( iCol>=p->iCol ); 343 + if( iCol!=p->iCol ){ 344 + if( pHash->eDetail==FTS5_DETAIL_FULL ){ 345 + pPtr[p->nData++] = 0x01; 346 + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); 347 + p->iCol = iCol; 348 + p->iPos = 0; 349 + }else{ 350 + bNew = 1; 351 + p->iCol = iPos = iCol; 352 + } 353 + } 354 + 355 + /* Append the new position offset, if necessary */ 356 + if( bNew ){ 357 + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); 358 + p->iPos = iPos; 359 + } 302 360 } 303 - 304 - /* Append the new position offset */ 305 - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); 306 - p->iPos = iPos; 307 361 }else{ 308 362 /* This is a delete. Set the delete flag. */ 309 363 p->bDel = 1; 310 364 } 365 + 311 366 nIncr += p->nData; 312 - 313 367 *pHash->pnByte += nIncr; 314 368 return SQLITE_OK; 315 369 } 316 370 317 371 318 372 /* 319 373 ** Arguments pLeft and pRight point to linked-lists of hash-entry objects, ................................................................................ 419 473 Fts5HashEntry *p; 420 474 421 475 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ 422 476 if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; 423 477 } 424 478 425 479 if( p ){ 426 - fts5HashAddPoslistSize(p); 480 + fts5HashAddPoslistSize(pHash, p); 427 481 *ppDoclist = (const u8*)&p->zKey[nTerm+1]; 428 482 *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); 429 483 }else{ 430 484 *ppDoclist = 0; 431 485 *pnDoclist = 0; 432 486 } 433 487 ................................................................................ 455 509 const char **pzTerm, /* OUT: term (nul-terminated) */ 456 510 const u8 **ppDoclist, /* OUT: pointer to doclist */ 457 511 int *pnDoclist /* OUT: size of doclist in bytes */ 458 512 ){ 459 513 Fts5HashEntry *p; 460 514 if( (p = pHash->pScan) ){ 461 515 int nTerm = (int)strlen(p->zKey); 462 - fts5HashAddPoslistSize(p); 516 + fts5HashAddPoslistSize(pHash, p); 463 517 *pzTerm = p->zKey; 464 518 *ppDoclist = (const u8*)&p->zKey[nTerm+1]; 465 519 *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); 466 520 }else{ 467 521 *pzTerm = 0; 468 522 *ppDoclist = 0; 469 523 *pnDoclist = 0; 470 524 } 471 525 } 472 526
Changes to ext/fts5/fts5_index.c.
429 429 Fts5StructureSegment *pSeg; /* Segment to iterate through */ 430 430 int flags; /* Mask of configuration flags */ 431 431 int iLeafPgno; /* Current leaf page number */ 432 432 Fts5Data *pLeaf; /* Current leaf data */ 433 433 Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ 434 434 int iLeafOffset; /* Byte offset within current leaf */ 435 435 436 + /* Next method */ 437 + void (*xNext)(Fts5Index*, Fts5SegIter*, int*); 438 + 436 439 /* The page and offset from which the current term was read. The offset 437 440 ** is the offset of the first rowid in the current doclist. */ 438 441 int iTermLeafPgno; 439 442 int iTermLeafOffset; 440 443 441 444 int iPgidxOff; /* Next offset in pgidx */ 442 445 int iEndofDoclist; ................................................................................ 448 451 449 452 Fts5DlidxIter *pDlidx; /* If there is a doclist-index */ 450 453 451 454 /* Variables populated based on current entry. */ 452 455 Fts5Buffer term; /* Current term */ 453 456 i64 iRowid; /* Current rowid */ 454 457 int nPos; /* Number of bytes in current position list */ 455 - int bDel; /* True if the delete flag is set */ 458 + u8 bDel; /* True if the delete flag is set */ 456 459 }; 457 460 458 461 /* 459 462 ** Argument is a pointer to an Fts5Data structure that contains a 460 463 ** leaf page. 461 464 */ 462 465 #define ASSERT_SZLEAF_OK(x) assert( \ 463 466 (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \ 464 467 ) 465 468 466 469 #define FTS5_SEGITER_ONETERM 0x01 467 470 #define FTS5_SEGITER_REVERSE 0x02 468 - 469 471 470 472 /* 471 473 ** Argument is a pointer to an Fts5Data structure that contains a leaf 472 474 ** page. This macro evaluates to true if the leaf contains no terms, or 473 475 ** false if it contains at least one term. 474 476 */ 475 477 #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn) ................................................................................ 1488 1490 ** 1489 1491 ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the 1490 1492 ** position list content (if any). 1491 1493 */ 1492 1494 static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ 1493 1495 if( p->rc==SQLITE_OK ){ 1494 1496 int iOff = pIter->iLeafOffset; /* Offset to read at */ 1495 - int nSz; 1496 1497 ASSERT_SZLEAF_OK(pIter->pLeaf); 1497 - fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); 1498 - pIter->bDel = (nSz & 0x0001); 1499 - pIter->nPos = nSz>>1; 1498 + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ 1499 + int iEod = MIN(pIter->iEndofDoclist, pIter->pLeaf->szLeaf); 1500 + pIter->bDel = 0; 1501 + pIter->nPos = 1; 1502 + if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ 1503 + pIter->bDel = 1; 1504 + iOff++; 1505 + if( iOff<iEod && pIter->pLeaf->p[iOff]==0 ){ 1506 + pIter->nPos = 1; 1507 + iOff++; 1508 + }else{ 1509 + pIter->nPos = 0; 1510 + } 1511 + } 1512 + }else{ 1513 + int nSz; 1514 + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); 1515 + pIter->bDel = (nSz & 0x0001); 1516 + pIter->nPos = nSz>>1; 1517 + assert_nc( pIter->nPos>=0 ); 1518 + } 1500 1519 pIter->iLeafOffset = iOff; 1501 - assert_nc( pIter->nPos>=0 ); 1502 1520 } 1503 1521 } 1504 1522 1505 1523 static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ 1506 1524 u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ 1507 1525 int iOff = pIter->iLeafOffset; 1508 1526 ................................................................................ 1554 1572 int nExtra; 1555 1573 pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra); 1556 1574 pIter->iEndofDoclist += nExtra; 1557 1575 } 1558 1576 1559 1577 fts5SegIterLoadRowid(p, pIter); 1560 1578 } 1579 + 1580 +static void fts5SegIterNext(Fts5Index*, Fts5SegIter*, int*); 1581 +static void fts5SegIterNext_Reverse(Fts5Index*, Fts5SegIter*, int*); 1582 +static void fts5SegIterNext_None(Fts5Index*, Fts5SegIter*, int*); 1583 + 1584 +static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ 1585 + if( pIter->flags & FTS5_SEGITER_REVERSE ){ 1586 + pIter->xNext = fts5SegIterNext_Reverse; 1587 + }else if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ 1588 + pIter->xNext = fts5SegIterNext_None; 1589 + }else{ 1590 + pIter->xNext = fts5SegIterNext; 1591 + } 1592 +} 1561 1593 1562 1594 /* 1563 1595 ** Initialize the iterator object pIter to iterate through the entries in 1564 1596 ** segment pSeg. The iterator is left pointing to the first entry when 1565 1597 ** this function returns. 1566 1598 ** 1567 1599 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ................................................................................ 1580 1612 ** at EOF already. */ 1581 1613 assert( pIter->pLeaf==0 ); 1582 1614 return; 1583 1615 } 1584 1616 1585 1617 if( p->rc==SQLITE_OK ){ 1586 1618 memset(pIter, 0, sizeof(*pIter)); 1619 + fts5SegIterSetNext(p, pIter); 1587 1620 pIter->pSeg = pSeg; 1588 1621 pIter->iLeafPgno = pSeg->pgnoFirst-1; 1589 1622 fts5SegIterNextPage(p, pIter); 1590 1623 } 1591 1624 1592 1625 if( p->rc==SQLITE_OK ){ 1593 1626 pIter->iLeafOffset = 4; ................................................................................ 1611 1644 ** This function advances the iterator so that it points to the last 1612 1645 ** relevant rowid on the page and, if necessary, initializes the 1613 1646 ** aRowidOffset[] and iRowidOffset variables. At this point the iterator 1614 1647 ** is in its regular state - Fts5SegIter.iLeafOffset points to the first 1615 1648 ** byte of the position list content associated with said rowid. 1616 1649 */ 1617 1650 static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ 1651 + int eDetail = p->pConfig->eDetail; 1618 1652 int n = pIter->pLeaf->szLeaf; 1619 1653 int i = pIter->iLeafOffset; 1620 1654 u8 *a = pIter->pLeaf->p; 1621 1655 int iRowidOffset = 0; 1622 1656 1623 1657 if( n>pIter->iEndofDoclist ){ 1624 1658 n = pIter->iEndofDoclist; 1625 1659 } 1626 1660 1627 1661 ASSERT_SZLEAF_OK(pIter->pLeaf); 1628 1662 while( 1 ){ 1629 1663 i64 iDelta = 0; 1630 - int nPos; 1631 - int bDummy; 1632 1664 1633 - i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); 1634 - i += nPos; 1665 + if( eDetail==FTS5_DETAIL_NONE ){ 1666 + /* todo */ 1667 + if( i<n && a[i]==0 ){ 1668 + i++; 1669 + if( i<n && a[i]==0 ) i++; 1670 + } 1671 + }else{ 1672 + int nPos; 1673 + int bDummy; 1674 + i += fts5GetPoslistSize(&a[i], &nPos, &bDummy); 1675 + i += nPos; 1676 + } 1635 1677 if( i>=n ) break; 1636 1678 i += fts5GetVarint(&a[i], (u64*)&iDelta); 1637 1679 pIter->iRowid += iDelta; 1638 1680 1681 + /* If necessary, grow the pIter->aRowidOffset[] array. */ 1639 1682 if( iRowidOffset>=pIter->nRowidOffset ){ 1640 1683 int nNew = pIter->nRowidOffset + 8; 1641 1684 int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); 1642 1685 if( aNew==0 ){ 1643 1686 p->rc = SQLITE_NOMEM; 1644 1687 break; 1645 1688 } ................................................................................ 1709 1752 ** points to a delete marker. A delete marker is an entry with a 0 byte 1710 1753 ** position-list. 1711 1754 */ 1712 1755 static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){ 1713 1756 Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; 1714 1757 return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); 1715 1758 } 1759 + 1760 +/* 1761 +** Advance iterator pIter to the next entry. 1762 +** 1763 +** This version of fts5SegIterNext() is only used by reverse iterators. 1764 +*/ 1765 +static void fts5SegIterNext_Reverse( 1766 + Fts5Index *p, /* FTS5 backend object */ 1767 + Fts5SegIter *pIter, /* Iterator to advance */ 1768 + int *pbNewTerm /* OUT: Set for new term */ 1769 +){ 1770 + assert( pIter->flags & FTS5_SEGITER_REVERSE ); 1771 + assert( pIter->pNextLeaf==0 ); 1772 + if( pIter->iRowidOffset>0 ){ 1773 + u8 *a = pIter->pLeaf->p; 1774 + int iOff; 1775 + i64 iDelta; 1776 + 1777 + pIter->iRowidOffset--; 1778 + pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; 1779 + fts5SegIterLoadNPos(p, pIter); 1780 + iOff = pIter->iLeafOffset; 1781 + if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ 1782 + iOff += pIter->nPos; 1783 + } 1784 + fts5GetVarint(&a[iOff], (u64*)&iDelta); 1785 + pIter->iRowid -= iDelta; 1786 + }else{ 1787 + fts5SegIterReverseNewPage(p, pIter); 1788 + } 1789 +} 1790 + 1791 +/* 1792 +** Advance iterator pIter to the next entry. 1793 +** 1794 +** This version of fts5SegIterNext() is only used if detail=none and the 1795 +** iterator is not a reverse direction iterator. 1796 +*/ 1797 +static void fts5SegIterNext_None( 1798 + Fts5Index *p, /* FTS5 backend object */ 1799 + Fts5SegIter *pIter, /* Iterator to advance */ 1800 + int *pbNewTerm /* OUT: Set for new term */ 1801 +){ 1802 + int iOff; 1803 + 1804 + assert( p->rc==SQLITE_OK ); 1805 + assert( (pIter->flags & FTS5_SEGITER_REVERSE)==0 ); 1806 + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE ); 1807 + 1808 + ASSERT_SZLEAF_OK(pIter->pLeaf); 1809 + iOff = pIter->iLeafOffset; 1810 + 1811 + /* Next entry is on the next page */ 1812 + if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ 1813 + fts5SegIterNextPage(p, pIter); 1814 + if( p->rc || pIter->pLeaf==0 ) return; 1815 + pIter->iRowid = 0; 1816 + iOff = 4; 1817 + } 1818 + 1819 + if( iOff<pIter->iEndofDoclist ){ 1820 + /* Next entry is on the current page */ 1821 + i64 iDelta; 1822 + iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); 1823 + pIter->iLeafOffset = iOff; 1824 + pIter->iRowid += iDelta; 1825 + }else if( (pIter->flags & FTS5_SEGITER_ONETERM)==0 ){ 1826 + if( pIter->pSeg ){ 1827 + int nKeep = 0; 1828 + if( iOff!=fts5LeafFirstTermOff(pIter->pLeaf) ){ 1829 + iOff += fts5GetVarint32(&pIter->pLeaf->p[iOff], nKeep); 1830 + } 1831 + pIter->iLeafOffset = iOff; 1832 + fts5SegIterLoadTerm(p, pIter, nKeep); 1833 + }else{ 1834 + const u8 *pList = 0; 1835 + const char *zTerm = 0; 1836 + int nList; 1837 + sqlite3Fts5HashScanNext(p->pHash); 1838 + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); 1839 + if( pList==0 ) goto next_none_eof; 1840 + pIter->pLeaf->p = (u8*)pList; 1841 + pIter->pLeaf->nn = nList; 1842 + pIter->pLeaf->szLeaf = nList; 1843 + pIter->iEndofDoclist = nList; 1844 + sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); 1845 + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); 1846 + } 1847 + 1848 + if( pbNewTerm ) *pbNewTerm = 1; 1849 + }else{ 1850 + goto next_none_eof; 1851 + } 1852 + 1853 + fts5SegIterLoadNPos(p, pIter); 1854 + 1855 + return; 1856 + next_none_eof: 1857 + fts5DataRelease(pIter->pLeaf); 1858 + pIter->pLeaf = 0; 1859 +} 1860 + 1716 1861 1717 1862 /* 1718 1863 ** Advance iterator pIter to the next entry. 1719 1864 ** 1720 1865 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. It 1721 1866 ** is not considered an error if the iterator reaches EOF. If an error has 1722 1867 ** already occurred when this function is called, it is a no-op. 1723 1868 */ 1724 1869 static void fts5SegIterNext( 1725 1870 Fts5Index *p, /* FTS5 backend object */ 1726 1871 Fts5SegIter *pIter, /* Iterator to advance */ 1727 1872 int *pbNewTerm /* OUT: Set for new term */ 1728 1873 ){ 1874 + Fts5Data *pLeaf = pIter->pLeaf; 1875 + int iOff; 1876 + int bNewTerm = 0; 1877 + int nKeep = 0; 1878 + 1729 1879 assert( pbNewTerm==0 || *pbNewTerm==0 ); 1730 - if( p->rc==SQLITE_OK ){ 1731 - if( pIter->flags & FTS5_SEGITER_REVERSE ){ 1732 - assert( pIter->pNextLeaf==0 ); 1733 - if( pIter->iRowidOffset>0 ){ 1734 - u8 *a = pIter->pLeaf->p; 1735 - int iOff; 1736 - int nPos; 1737 - int bDummy; 1738 - i64 iDelta; 1739 - 1740 - pIter->iRowidOffset--; 1741 - pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset]; 1742 - iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDummy); 1743 - iOff += nPos; 1744 - fts5GetVarint(&a[iOff], (u64*)&iDelta); 1745 - pIter->iRowid -= iDelta; 1746 - fts5SegIterLoadNPos(p, pIter); 1747 - }else{ 1748 - fts5SegIterReverseNewPage(p, pIter); 1749 - } 1750 - }else{ 1751 - Fts5Data *pLeaf = pIter->pLeaf; 1752 - int iOff; 1753 - int bNewTerm = 0; 1754 - int nKeep = 0; 1755 - 1756 - /* Search for the end of the position list within the current page. */ 1757 - u8 *a = pLeaf->p; 1758 - int n = pLeaf->szLeaf; 1759 - 1880 + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); 1881 + 1882 + /* Search for the end of the position list within the current page. */ 1883 + u8 *a = pLeaf->p; 1884 + int n = pLeaf->szLeaf; 1885 + 1886 + ASSERT_SZLEAF_OK(pLeaf); 1887 + iOff = pIter->iLeafOffset + pIter->nPos; 1888 + 1889 + if( iOff<n ){ 1890 + /* The next entry is on the current page. */ 1891 + assert_nc( iOff<=pIter->iEndofDoclist ); 1892 + if( iOff>=pIter->iEndofDoclist ){ 1893 + bNewTerm = 1; 1894 + if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ 1895 + iOff += fts5GetVarint32(&a[iOff], nKeep); 1896 + } 1897 + }else{ 1898 + u64 iDelta; 1899 + iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); 1900 + pIter->iRowid += iDelta; 1901 + assert_nc( iDelta>0 ); 1902 + } 1903 + pIter->iLeafOffset = iOff; 1904 + 1905 + }else if( pIter->pSeg==0 ){ 1906 + const u8 *pList = 0; 1907 + const char *zTerm = 0; 1908 + int nList = 0; 1909 + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); 1910 + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ 1911 + sqlite3Fts5HashScanNext(p->pHash); 1912 + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); 1913 + } 1914 + if( pList==0 ){ 1915 + fts5DataRelease(pIter->pLeaf); 1916 + pIter->pLeaf = 0; 1917 + }else{ 1918 + pIter->pLeaf->p = (u8*)pList; 1919 + pIter->pLeaf->nn = nList; 1920 + pIter->pLeaf->szLeaf = nList; 1921 + pIter->iEndofDoclist = nList+1; 1922 + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), 1923 + (u8*)zTerm); 1924 + pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); 1925 + *pbNewTerm = 1; 1926 + } 1927 + }else{ 1928 + iOff = 0; 1929 + /* Next entry is not on the current page */ 1930 + while( iOff==0 ){ 1931 + fts5SegIterNextPage(p, pIter); 1932 + pLeaf = pIter->pLeaf; 1933 + if( pLeaf==0 ) break; 1760 1934 ASSERT_SZLEAF_OK(pLeaf); 1761 - iOff = pIter->iLeafOffset + pIter->nPos; 1762 - 1763 - if( iOff<n ){ 1764 - /* The next entry is on the current page. */ 1765 - assert_nc( iOff<=pIter->iEndofDoclist ); 1766 - if( iOff>=pIter->iEndofDoclist ){ 1767 - bNewTerm = 1; 1768 - if( iOff!=fts5LeafFirstTermOff(pLeaf) ){ 1769 - iOff += fts5GetVarint32(&a[iOff], nKeep); 1770 - } 1771 - }else{ 1772 - u64 iDelta; 1773 - iOff += sqlite3Fts5GetVarint(&a[iOff], &iDelta); 1774 - pIter->iRowid += iDelta; 1775 - assert_nc( iDelta>0 ); 1776 - } 1935 + if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ 1936 + iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); 1777 1937 pIter->iLeafOffset = iOff; 1778 1938 1779 - }else if( pIter->pSeg==0 ){ 1780 - const u8 *pList = 0; 1781 - const char *zTerm = 0; 1782 - int nList = 0; 1783 - assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); 1784 - if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ 1785 - sqlite3Fts5HashScanNext(p->pHash); 1786 - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); 1787 - } 1788 - if( pList==0 ){ 1789 - fts5DataRelease(pIter->pLeaf); 1790 - pIter->pLeaf = 0; 1791 - }else{ 1792 - pIter->pLeaf->p = (u8*)pList; 1793 - pIter->pLeaf->nn = nList; 1794 - pIter->pLeaf->szLeaf = nList; 1795 - pIter->iEndofDoclist = nList+1; 1796 - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), 1797 - (u8*)zTerm); 1798 - pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); 1799 - *pbNewTerm = 1; 1800 - } 1801 - }else{ 1802 - iOff = 0; 1803 - /* Next entry is not on the current page */ 1804 - while( iOff==0 ){ 1805 - fts5SegIterNextPage(p, pIter); 1806 - pLeaf = pIter->pLeaf; 1807 - if( pLeaf==0 ) break; 1808 - ASSERT_SZLEAF_OK(pLeaf); 1809 - if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){ 1810 - iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid); 1811 - pIter->iLeafOffset = iOff; 1812 - 1813 - if( pLeaf->nn>pLeaf->szLeaf ){ 1814 - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1815 - &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist 1939 + if( pLeaf->nn>pLeaf->szLeaf ){ 1940 + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1941 + &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist 1816 1942 ); 1817 - } 1943 + } 1818 1944 1819 - } 1820 - else if( pLeaf->nn>pLeaf->szLeaf ){ 1821 - pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1822 - &pLeaf->p[pLeaf->szLeaf], iOff 1945 + } 1946 + else if( pLeaf->nn>pLeaf->szLeaf ){ 1947 + pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( 1948 + &pLeaf->p[pLeaf->szLeaf], iOff 1823 1949 ); 1824 - pIter->iLeafOffset = iOff; 1825 - pIter->iEndofDoclist = iOff; 1826 - bNewTerm = 1; 1827 - } 1828 - if( iOff>=pLeaf->szLeaf ){ 1829 - p->rc = FTS5_CORRUPT; 1830 - return; 1831 - } 1832 - } 1833 - } 1834 - 1835 - /* Check if the iterator is now at EOF. If so, return early. */ 1836 - if( pIter->pLeaf ){ 1837 - if( bNewTerm ){ 1838 - if( pIter->flags & FTS5_SEGITER_ONETERM ){ 1839 - fts5DataRelease(pIter->pLeaf); 1840 - pIter->pLeaf = 0; 1841 - }else{ 1842 - fts5SegIterLoadTerm(p, pIter, nKeep); 1843 - fts5SegIterLoadNPos(p, pIter); 1844 - if( pbNewTerm ) *pbNewTerm = 1; 1845 - } 1846 - }else{ 1847 - /* The following could be done by calling fts5SegIterLoadNPos(). But 1848 - ** this block is particularly performance critical, so equivalent 1849 - ** code is inlined. */ 1850 - int nSz; 1851 - assert( p->rc==SQLITE_OK ); 1852 - fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); 1853 - pIter->bDel = (nSz & 0x0001); 1854 - pIter->nPos = nSz>>1; 1855 - assert_nc( pIter->nPos>=0 ); 1856 - } 1857 - } 1950 + pIter->iLeafOffset = iOff; 1951 + pIter->iEndofDoclist = iOff; 1952 + bNewTerm = 1; 1953 + } 1954 + assert_nc( iOff<pLeaf->szLeaf ); 1955 + if( iOff>pLeaf->szLeaf ){ 1956 + p->rc = FTS5_CORRUPT; 1957 + return; 1958 + } 1959 + } 1960 + } 1961 + 1962 + /* Check if the iterator is now at EOF. If so, return early. */ 1963 + if( pIter->pLeaf ){ 1964 + if( bNewTerm ){ 1965 + if( pIter->flags & FTS5_SEGITER_ONETERM ){ 1966 + fts5DataRelease(pIter->pLeaf); 1967 + pIter->pLeaf = 0; 1968 + }else{ 1969 + fts5SegIterLoadTerm(p, pIter, nKeep); 1970 + fts5SegIterLoadNPos(p, pIter); 1971 + if( pbNewTerm ) *pbNewTerm = 1; 1972 + } 1973 + }else{ 1974 + /* The following could be done by calling fts5SegIterLoadNPos(). But 1975 + ** this block is particularly performance critical, so equivalent 1976 + ** code is inlined. 1977 + ** 1978 + ** Later: Switched back to fts5SegIterLoadNPos() because it supports 1979 + ** detail=none mode. Not ideal. 1980 + */ 1981 + int nSz; 1982 + assert( p->rc==SQLITE_OK ); 1983 + fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); 1984 + pIter->bDel = (nSz & 0x0001); 1985 + pIter->nPos = nSz>>1; 1986 + assert_nc( pIter->nPos>=0 ); 1858 1987 } 1859 1988 } 1860 1989 } 1861 1990 1862 1991 #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } 1992 + 1993 +#define fts5IndexSkipVarint(a, iOff) { \ 1994 + int iEnd = iOff+9; \ 1995 + while( (a[iOff++] & 0x80) && iOff<iEnd ); \ 1996 +} 1863 1997 1864 1998 /* 1865 1999 ** Iterator pIter currently points to the first rowid in a doclist. This 1866 2000 ** function sets the iterator up so that iterates in reverse order through 1867 2001 ** the doclist. 1868 2002 */ 1869 2003 static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ................................................................................ 1877 2011 pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); 1878 2012 }else{ 1879 2013 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ 1880 2014 1881 2015 /* Currently, Fts5SegIter.iLeafOffset points to the first byte of 1882 2016 ** position-list content for the current rowid. Back it up so that it 1883 2017 ** points to the start of the position-list size field. */ 1884 - pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); 2018 + int iPoslist; 2019 + if( pIter->iTermLeafPgno==pIter->iLeafPgno ){ 2020 + iPoslist = pIter->iTermLeafOffset; 2021 + }else{ 2022 + iPoslist = 4; 2023 + } 2024 + fts5IndexSkipVarint(pLeaf->p, iPoslist); 2025 + assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==( 2026 + pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel) 2027 + )); 2028 + pIter->iLeafOffset = iPoslist; 1885 2029 1886 2030 /* If this condition is true then the largest rowid for the current 1887 2031 ** term may not be stored on the current page. So search forward to 1888 2032 ** see where said rowid really is. */ 1889 2033 if( pIter->iEndofDoclist>=pLeaf->szLeaf ){ 1890 2034 int pgno; 1891 2035 Fts5StructureSegment *pSeg = pIter->pSeg; ................................................................................ 1961 2105 ){ 1962 2106 return; 1963 2107 } 1964 2108 1965 2109 pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); 1966 2110 } 1967 2111 1968 -#define fts5IndexSkipVarint(a, iOff) { \ 1969 - int iEnd = iOff+9; \ 1970 - while( (a[iOff++] & 0x80) && iOff<iEnd ); \ 1971 -} 1972 - 1973 2112 /* 1974 2113 ** The iterator object passed as the second argument currently contains 1975 2114 ** no valid values except for the Fts5SegIter.pLeaf member variable. This 1976 2115 ** function searches the leaf page for a term matching (pTerm/nTerm). 1977 2116 ** 1978 2117 ** If the specified term is found on the page, then the iterator is left 1979 2118 ** pointing to it. If argument bGe is zero and the term is not found, ................................................................................ 2168 2307 } 2169 2308 if( flags & FTS5INDEX_QUERY_DESC ){ 2170 2309 fts5SegIterReverse(p, pIter); 2171 2310 } 2172 2311 } 2173 2312 } 2174 2313 2314 + fts5SegIterSetNext(p, pIter); 2315 + 2175 2316 /* Either: 2176 2317 ** 2177 2318 ** 1) an error has occurred, or 2178 2319 ** 2) the iterator points to EOF, or 2179 2320 ** 3) the iterator points to an entry with term (pTerm/nTerm), or 2180 2321 ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points 2181 2322 ** to an entry with a term greater than or equal to (pTerm/nTerm). ................................................................................ 2225 2366 sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); 2226 2367 pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); 2227 2368 if( pLeaf==0 ) return; 2228 2369 pLeaf->p = (u8*)pList; 2229 2370 pLeaf->nn = pLeaf->szLeaf = nList; 2230 2371 pIter->pLeaf = pLeaf; 2231 2372 pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); 2232 - pIter->iEndofDoclist = pLeaf->nn+1; 2373 + pIter->iEndofDoclist = pLeaf->nn; 2233 2374 2234 2375 if( flags & FTS5INDEX_QUERY_DESC ){ 2235 2376 pIter->flags |= FTS5_SEGITER_REVERSE; 2236 2377 fts5SegIterReverseInitPage(p, pIter); 2237 2378 }else{ 2238 2379 fts5SegIterLoadNPos(p, pIter); 2239 2380 } 2240 2381 } 2382 + 2383 + fts5SegIterSetNext(p, pIter); 2241 2384 } 2242 2385 2243 2386 /* 2244 2387 ** Zero the iterator passed as the only argument. 2245 2388 */ 2246 2389 static void fts5SegIterClear(Fts5SegIter *pIter){ 2247 2390 fts5BufferFree(&pIter->term); ................................................................................ 2477 2620 pIter->iLeafPgno = iLeafPgno+1; 2478 2621 fts5SegIterReverseNewPage(p, pIter); 2479 2622 bMove = 0; 2480 2623 } 2481 2624 } 2482 2625 2483 2626 do{ 2484 - if( bMove ) fts5SegIterNext(p, pIter, 0); 2627 + if( bMove && p->rc==SQLITE_OK ) pIter->xNext(p, pIter, 0); 2485 2628 if( pIter->pLeaf==0 ) break; 2486 2629 if( bRev==0 && pIter->iRowid>=iMatch ) break; 2487 2630 if( bRev!=0 && pIter->iRowid<=iMatch ) break; 2488 2631 bMove = 1; 2489 2632 }while( p->rc==SQLITE_OK ); 2490 2633 } 2491 2634 ................................................................................ 2511 2654 int iChanged, /* Index of sub-iterator just advanced */ 2512 2655 int iMinset /* Minimum entry in aFirst[] to set */ 2513 2656 ){ 2514 2657 int i; 2515 2658 for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){ 2516 2659 int iEq; 2517 2660 if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){ 2518 - fts5SegIterNext(p, &pIter->aSeg[iEq], 0); 2661 + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; 2662 + assert( p->rc==SQLITE_OK ); 2663 + pSeg->xNext(p, pSeg, 0); 2519 2664 i = pIter->nSeg + iEq; 2520 2665 } 2521 2666 } 2522 2667 } 2523 2668 2524 2669 /* 2525 2670 ** Sub-iterator iChanged of iterator pIter has just been advanced. It still ................................................................................ 2598 2743 int iFirst = pIter->aFirst[1].iFirst; 2599 2744 int bNewTerm = 0; 2600 2745 Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; 2601 2746 assert( p->rc==SQLITE_OK ); 2602 2747 if( bUseFrom && pSeg->pDlidx ){ 2603 2748 fts5SegIterNextFrom(p, pSeg, iFrom); 2604 2749 }else{ 2605 - fts5SegIterNext(p, pSeg, &bNewTerm); 2750 + pSeg->xNext(p, pSeg, &bNewTerm); 2606 2751 } 2607 2752 2608 2753 if( pSeg->pLeaf==0 || bNewTerm 2609 2754 || fts5MultiIterAdvanceRowid(p, pIter, iFirst) 2610 2755 ){ 2611 2756 fts5MultiIterAdvanced(p, pIter, iFirst, 1); 2612 2757 fts5MultiIterSetEof(pIter); ................................................................................ 2626 2771 assert( pIter->bSkipEmpty ); 2627 2772 if( p->rc==SQLITE_OK ){ 2628 2773 do { 2629 2774 int iFirst = pIter->aFirst[1].iFirst; 2630 2775 Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; 2631 2776 int bNewTerm = 0; 2632 2777 2633 - fts5SegIterNext(p, pSeg, &bNewTerm); 2778 + assert( p->rc==SQLITE_OK ); 2779 + pSeg->xNext(p, pSeg, &bNewTerm); 2634 2780 if( pSeg->pLeaf==0 || bNewTerm 2635 2781 || fts5MultiIterAdvanceRowid(p, pIter, iFirst) 2636 2782 ){ 2637 2783 fts5MultiIterAdvanced(p, pIter, iFirst, 1); 2638 2784 fts5MultiIterSetEof(pIter); 2639 2785 *pbNewTerm = 1; 2640 2786 }else{ ................................................................................ 2746 2892 ** to the first entry in its segment. In this case initialize the 2747 2893 ** aFirst[] array. Or, if an error has occurred, free the iterator 2748 2894 ** object and set the output variable to NULL. */ 2749 2895 if( p->rc==SQLITE_OK ){ 2750 2896 for(iIter=pNew->nSeg-1; iIter>0; iIter--){ 2751 2897 int iEq; 2752 2898 if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ 2753 - fts5SegIterNext(p, &pNew->aSeg[iEq], 0); 2899 + Fts5SegIter *pSeg = &pNew->aSeg[iEq]; 2900 + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); 2754 2901 fts5MultiIterAdvanced(p, pNew, iEq, iIter); 2755 2902 } 2756 2903 } 2757 2904 fts5MultiIterSetEof(pNew); 2758 2905 fts5AssertMultiIterSetup(p, pNew); 2759 2906 2760 2907 if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ ................................................................................ 2796 2943 }else{ 2797 2944 fts5SegIterLoadNPos(p, pIter); 2798 2945 } 2799 2946 pData = 0; 2800 2947 }else{ 2801 2948 pNew->bEof = 1; 2802 2949 } 2950 + fts5SegIterSetNext(p, pIter); 2803 2951 2804 2952 *ppOut = pNew; 2805 2953 } 2806 2954 2807 2955 fts5DataRelease(pData); 2808 2956 } 2809 2957 ................................................................................ 2864 3012 ){ 2865 3013 int nRem = pSeg->nPos; /* Number of bytes still to come */ 2866 3014 Fts5Data *pData = 0; 2867 3015 u8 *pChunk = &pSeg->pLeaf->p[pSeg->iLeafOffset]; 2868 3016 int nChunk = MIN(nRem, pSeg->pLeaf->szLeaf - pSeg->iLeafOffset); 2869 3017 int pgno = pSeg->iLeafPgno; 2870 3018 int pgnoSave = 0; 3019 + 3020 + /* This function does notmwork with detail=none databases. */ 3021 + assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); 2871 3022 2872 3023 if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ 2873 3024 pgnoSave = pgno+1; 2874 3025 } 2875 3026 2876 3027 while( 1 ){ 2877 3028 xChunk(p, pCtx, pChunk, nChunk); ................................................................................ 3288 3439 3289 3440 /* 3290 3441 ** Append a rowid and position-list size field to the writers output. 3291 3442 */ 3292 3443 static void fts5WriteAppendRowid( 3293 3444 Fts5Index *p, 3294 3445 Fts5SegWriter *pWriter, 3295 - i64 iRowid, 3296 - int nPos 3446 + i64 iRowid 3297 3447 ){ 3298 3448 if( p->rc==SQLITE_OK ){ 3299 3449 Fts5PageWriter *pPage = &pWriter->writer; 3300 3450 3301 3451 if( (pPage->buf.n + pPage->pgidx.n)>=p->pConfig->pgsz ){ 3302 3452 fts5WriteFlushLeaf(p, pWriter); 3303 3453 } ................................................................................ 3316 3466 }else{ 3317 3467 assert( p->rc || iRowid>pWriter->iPrevRowid ); 3318 3468 fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); 3319 3469 } 3320 3470 pWriter->iPrevRowid = iRowid; 3321 3471 pWriter->bFirstRowidInDoclist = 0; 3322 3472 pWriter->bFirstRowidInPage = 0; 3323 - 3324 - fts5BufferAppendVarint(&p->rc, &pPage->buf, nPos); 3325 3473 } 3326 3474 } 3327 3475 3328 3476 static void fts5WriteAppendPoslistData( 3329 3477 Fts5Index *p, 3330 3478 Fts5SegWriter *pWriter, 3331 3479 const u8 *aData, ................................................................................ 3513 3661 Fts5IndexIter *pIter = 0; /* Iterator to read input data */ 3514 3662 int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ 3515 3663 int nInput; /* Number of input segments */ 3516 3664 Fts5SegWriter writer; /* Writer object */ 3517 3665 Fts5StructureSegment *pSeg; /* Output segment */ 3518 3666 Fts5Buffer term; 3519 3667 int bOldest; /* True if the output segment is the oldest */ 3668 + int eDetail = p->pConfig->eDetail; 3520 3669 3521 3670 assert( iLvl<pStruct->nLevel ); 3522 3671 assert( pLvl->nMerge<=pLvl->nSeg ); 3523 3672 3524 3673 memset(&writer, 0, sizeof(Fts5SegWriter)); 3525 3674 memset(&term, 0, sizeof(Fts5Buffer)); 3526 3675 if( pLvl->nMerge ){ ................................................................................ 3582 3731 /* This is a new term. Append a term to the output segment. */ 3583 3732 fts5WriteAppendTerm(p, &writer, nTerm, pTerm); 3584 3733 fts5BufferSet(&p->rc, &term, nTerm, pTerm); 3585 3734 } 3586 3735 3587 3736 /* Append the rowid to the output */ 3588 3737 /* WRITEPOSLISTSIZE */ 3589 - nPos = pSegIter->nPos*2 + pSegIter->bDel; 3590 - fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter), nPos); 3738 + fts5WriteAppendRowid(p, &writer, fts5MultiIterRowid(pIter)); 3591 3739 3592 - /* Append the position-list data to the output */ 3593 - fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); 3740 + if( eDetail==FTS5_DETAIL_NONE ){ 3741 + if( pSegIter->bDel ){ 3742 + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); 3743 + if( pSegIter->nPos>0 ){ 3744 + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, 0); 3745 + } 3746 + } 3747 + }else{ 3748 + /* Append the position-list data to the output */ 3749 + nPos = pSegIter->nPos*2 + pSegIter->bDel; 3750 + fts5BufferAppendVarint(&p->rc, &writer.writer.buf, nPos); 3751 + fts5ChunkIterate(p, pSegIter, (void*)&writer, fts5MergeChunkCallback); 3752 + } 3594 3753 } 3595 3754 3596 3755 /* Flush the last leaf page to disk. Set the output segment b-tree height 3597 3756 ** and last leaf page number at the same time. */ 3598 3757 fts5WriteFinish(p, &writer, &pSeg->pgnoLast); 3599 3758 3600 3759 if( fts5MultiIterEof(p, pIter) ){ ................................................................................ 3774 3933 /* Obtain a reference to the index structure and allocate a new segment-id 3775 3934 ** for the new level-0 segment. */ 3776 3935 pStruct = fts5StructureRead(p); 3777 3936 iSegid = fts5AllocateSegid(p, pStruct); 3778 3937 3779 3938 if( iSegid ){ 3780 3939 const int pgsz = p->pConfig->pgsz; 3781 - 3940 + int eDetail = p->pConfig->eDetail; 3782 3941 Fts5StructureSegment *pSeg; /* New segment within pStruct */ 3783 3942 Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ 3784 3943 Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ 3785 3944 3786 3945 Fts5SegWriter writer; 3787 3946 fts5WriteInit(p, &writer, iSegid); 3788 3947 ................................................................................ 3817 3976 i64 iDelta = 0; 3818 3977 int iOff = 0; 3819 3978 3820 3979 /* The entire doclist will not fit on this leaf. The following 3821 3980 ** loop iterates through the poslists that make up the current 3822 3981 ** doclist. */ 3823 3982 while( p->rc==SQLITE_OK && iOff<nDoclist ){ 3824 - int nPos; 3825 - int nCopy; 3826 - int bDummy; 3827 3983 iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); 3828 - nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); 3829 - nCopy += nPos; 3830 3984 iRowid += iDelta; 3831 3985 3832 3986 if( writer.bFirstRowidInPage ){ 3833 3987 fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ 3834 3988 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); 3835 3989 writer.bFirstRowidInPage = 0; 3836 3990 fts5WriteDlidxAppend(p, &writer, iRowid); 3837 3991 }else{ 3838 3992 pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); 3839 3993 } 3840 3994 assert( pBuf->n<=pBuf->nSpace ); 3841 3995 3842 - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ 3843 - /* The entire poslist will fit on the current leaf. So copy 3844 - ** it in one go. */ 3845 - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); 3996 + if( eDetail==FTS5_DETAIL_NONE ){ 3997 + if( iOff<nDoclist && pDoclist[iOff]==0 ){ 3998 + pBuf->p[pBuf->n++] = 0; 3999 + iOff++; 4000 + if( iOff<nDoclist && pDoclist[iOff]==0 ){ 4001 + pBuf->p[pBuf->n++] = 0; 4002 + iOff++; 4003 + } 4004 + } 4005 + if( (pBuf->n + pPgidx->n)>=pgsz ){ 4006 + fts5WriteFlushLeaf(p, &writer); 4007 + } 3846 4008 }else{ 3847 - /* The entire poslist will not fit on this leaf. So it needs 3848 - ** to be broken into sections. The only qualification being 3849 - ** that each varint must be stored contiguously. */ 3850 - const u8 *pPoslist = &pDoclist[iOff]; 3851 - int iPos = 0; 3852 - while( p->rc==SQLITE_OK ){ 3853 - int nSpace = pgsz - pBuf->n - pPgidx->n; 3854 - int n = 0; 3855 - if( (nCopy - iPos)<=nSpace ){ 3856 - n = nCopy - iPos; 3857 - }else{ 3858 - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); 4009 + int bDummy; 4010 + int nPos; 4011 + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); 4012 + nCopy += nPos; 4013 + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ 4014 + /* The entire poslist will fit on the current leaf. So copy 4015 + ** it in one go. */ 4016 + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); 4017 + }else{ 4018 + /* The entire poslist will not fit on this leaf. So it needs 4019 + ** to be broken into sections. The only qualification being 4020 + ** that each varint must be stored contiguously. */ 4021 + const u8 *pPoslist = &pDoclist[iOff]; 4022 + int iPos = 0; 4023 + while( p->rc==SQLITE_OK ){ 4024 + int nSpace = pgsz - pBuf->n - pPgidx->n; 4025 + int n = 0; 4026 + if( (nCopy - iPos)<=nSpace ){ 4027 + n = nCopy - iPos; 4028 + }else{ 4029 + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); 4030 + } 4031 + assert( n>0 ); 4032 + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); 4033 + iPos += n; 4034 + if( (pBuf->n + pPgidx->n)>=pgsz ){ 4035 + fts5WriteFlushLeaf(p, &writer); 4036 + } 4037 + if( iPos>=nCopy ) break; 3859 4038 } 3860 - assert( n>0 ); 3861 - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); 3862 - iPos += n; 3863 - if( (pBuf->n + pPgidx->n)>=pgsz ){ 3864 - fts5WriteFlushLeaf(p, &writer); 3865 - } 3866 - if( iPos>=nCopy ) break; 3867 4039 } 4040 + iOff += nCopy; 3868 4041 } 3869 - iOff += nCopy; 3870 4042 } 3871 4043 } 3872 4044 3873 4045 /* TODO2: Doclist terminator written here. */ 3874 4046 /* pBuf->p[pBuf->n++] = '\0'; */ 3875 4047 assert( pBuf->n<=pBuf->nSpace ); 3876 4048 sqlite3Fts5HashScanNext(pHash); ................................................................................ 3997 4169 typedef struct PoslistCallbackCtx PoslistCallbackCtx; 3998 4170 struct PoslistCallbackCtx { 3999 4171 Fts5Buffer *pBuf; /* Append to this buffer */ 4000 4172 Fts5Colset *pColset; /* Restrict matches to this column */ 4001 4173 int eState; /* See above */ 4002 4174 }; 4003 4175 4176 +typedef struct PoslistOffsetsCtx PoslistOffsetsCtx; 4177 +struct PoslistOffsetsCtx { 4178 + Fts5Buffer *pBuf; /* Append to this buffer */ 4179 + Fts5Colset *pColset; /* Restrict matches to this column */ 4180 + int iRead; 4181 + int iWrite; 4182 +}; 4183 + 4004 4184 /* 4005 4185 ** TODO: Make this more efficient! 4006 4186 */ 4007 4187 static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ 4008 4188 int i; 4009 4189 for(i=0; i<pColset->nCol; i++){ 4010 4190 if( pColset->aiCol[i]==iCol ) return 1; 4011 4191 } 4012 4192 return 0; 4013 4193 } 4194 + 4195 +static void fts5PoslistOffsetsCallback( 4196 + Fts5Index *p, 4197 + void *pContext, 4198 + const u8 *pChunk, int nChunk 4199 +){ 4200 + PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; 4201 + assert_nc( nChunk>=0 ); 4202 + if( nChunk>0 ){ 4203 + int i = 0; 4204 + while( i<nChunk ){ 4205 + int iVal; 4206 + i += fts5GetVarint32(&pChunk[i], iVal); 4207 + iVal += pCtx->iRead - 2; 4208 + pCtx->iRead = iVal; 4209 + if( fts5IndexColsetTest(pCtx->pColset, iVal) ){ 4210 + fts5BufferSafeAppendVarint(pCtx->pBuf, iVal + 2 - pCtx->iWrite); 4211 + pCtx->iWrite = iVal; 4212 + } 4213 + } 4214 + } 4215 +} 4014 4216 4015 4217 static void fts5PoslistFilterCallback( 4016 4218 Fts5Index *p, 4017 4219 void *pContext, 4018 4220 const u8 *pChunk, int nChunk 4019 4221 ){ 4020 4222 PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; ................................................................................ 4075 4277 Fts5Colset *pColset, 4076 4278 Fts5Buffer *pBuf 4077 4279 ){ 4078 4280 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ 4079 4281 if( pColset==0 ){ 4080 4282 fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); 4081 4283 }else{ 4082 - PoslistCallbackCtx sCtx; 4083 - sCtx.pBuf = pBuf; 4084 - sCtx.pColset = pColset; 4085 - sCtx.eState = fts5IndexColsetTest(pColset, 0); 4086 - assert( sCtx.eState==0 || sCtx.eState==1 ); 4087 - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); 4284 + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ 4285 + PoslistCallbackCtx sCtx; 4286 + sCtx.pBuf = pBuf; 4287 + sCtx.pColset = pColset; 4288 + sCtx.eState = fts5IndexColsetTest(pColset, 0); 4289 + assert( sCtx.eState==0 || sCtx.eState==1 ); 4290 + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); 4291 + }else{ 4292 + PoslistOffsetsCtx sCtx; 4293 + memset(&sCtx, 0, sizeof(sCtx)); 4294 + sCtx.pBuf = pBuf; 4295 + sCtx.pColset = pColset; 4296 + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); 4297 + } 4088 4298 } 4089 4299 } 4090 4300 } 4091 4301 4092 4302 /* 4093 4303 ** IN/OUT parameter (*pa) points to a position list n bytes in size. If 4094 4304 ** the position list contains entries for column iCol, then (*pa) is set ................................................................................ 4123 4333 assert( (prev & 0x80)==0 ); 4124 4334 while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){ 4125 4335 prev = *p++; 4126 4336 } 4127 4337 return p - (*pa); 4128 4338 } 4129 4339 4340 +static int fts5AppendRowid( 4341 + Fts5Index *p, 4342 + i64 iDelta, 4343 + Fts5IndexIter *pMulti, 4344 + Fts5Colset *pColset, 4345 + Fts5Buffer *pBuf 4346 +){ 4347 + fts5BufferAppendVarint(&p->rc, pBuf, iDelta); 4348 + return 0; 4349 +} 4130 4350 4131 4351 /* 4132 4352 ** Iterator pMulti currently points to a valid entry (not EOF). This 4133 4353 ** function appends the following to buffer pBuf: 4134 4354 ** 4135 4355 ** * The varint iDelta, and 4136 4356 ** * the position list that currently points to, including the size field. ................................................................................ 4150 4370 Fts5Buffer *pBuf 4151 4371 ){ 4152 4372 if( p->rc==SQLITE_OK ){ 4153 4373 Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; 4154 4374 assert( fts5MultiIterEof(p, pMulti)==0 ); 4155 4375 assert( pSeg->nPos>0 ); 4156 4376 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ 4157 - 4158 - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 4377 + if( p->pConfig->eDetail==FTS5_DETAIL_FULL 4378 + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 4159 4379 && (pColset==0 || pColset->nCol==1) 4160 4380 ){ 4161 4381 const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; 4162 4382 int nPos; 4163 4383 if( pColset ){ 4164 4384 nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); 4165 4385 if( nPos==0 ) return 1; ................................................................................ 4196 4416 int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); 4197 4417 while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } 4198 4418 sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); 4199 4419 } 4200 4420 } 4201 4421 } 4202 4422 } 4203 - 4204 4423 } 4205 4424 } 4206 4425 4207 4426 return 0; 4208 4427 } 4428 + 4209 4429 4210 4430 static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ 4211 4431 u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; 4212 4432 4213 4433 assert( pIter->aPoslist ); 4214 4434 if( p>=pIter->aEof ){ 4215 4435 pIter->aPoslist = 0; ................................................................................ 4262 4482 #endif 4263 4483 4264 4484 #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ 4265 4485 assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ 4266 4486 fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ 4267 4487 (iLastRowid) = (iRowid); \ 4268 4488 } 4489 + 4490 +/* 4491 +** Swap the contents of buffer *p1 with that of *p2. 4492 +*/ 4493 +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ 4494 + Fts5Buffer tmp = *p1; 4495 + *p1 = *p2; 4496 + *p2 = tmp; 4497 +} 4498 + 4499 +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ 4500 + int i = *piOff; 4501 + if( i>=pBuf->n ){ 4502 + *piOff = -1; 4503 + }else{ 4504 + u64 iVal; 4505 + *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); 4506 + *piRowid += iVal; 4507 + } 4508 +} 4509 + 4510 +/* 4511 +** This is the equivalent of fts5MergePrefixLists() for detail=none mode. 4512 +** In this case the buffers consist of a delta-encoded list of rowids only. 4513 +*/ 4514 +static void fts5MergeRowidLists( 4515 + Fts5Index *p, /* FTS5 backend object */ 4516 + Fts5Buffer *p1, /* First list to merge */ 4517 + Fts5Buffer *p2 /* Second list to merge */ 4518 +){ 4519 + int i1 = 0; 4520 + int i2 = 0; 4521 + i64 iRowid1 = 0; 4522 + i64 iRowid2 = 0; 4523 + i64 iOut = 0; 4524 + 4525 + Fts5Buffer out; 4526 + memset(&out, 0, sizeof(out)); 4527 + sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); 4528 + if( p->rc ) return; 4529 + 4530 + fts5NextRowid(p1, &i1, &iRowid1); 4531 + fts5NextRowid(p2, &i2, &iRowid2); 4532 + while( i1>=0 || i2>=0 ){ 4533 + if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){ 4534 + assert( iOut==0 || iRowid1>iOut ); 4535 + fts5BufferSafeAppendVarint(&out, iRowid1 - iOut); 4536 + iOut = iRowid1; 4537 + fts5NextRowid(p1, &i1, &iRowid1); 4538 + }else{ 4539 + assert( iOut==0 || iRowid2>iOut ); 4540 + fts5BufferSafeAppendVarint(&out, iRowid2 - iOut); 4541 + iOut = iRowid2; 4542 + if( i1>=0 && iRowid1==iRowid2 ){ 4543 + fts5NextRowid(p1, &i1, &iRowid1); 4544 + } 4545 + fts5NextRowid(p2, &i2, &iRowid2); 4546 + } 4547 + } 4548 + 4549 + fts5BufferSwap(&out, p1); 4550 + fts5BufferFree(&out); 4551 +} 4269 4552 4270 4553 /* 4271 4554 ** Buffers p1 and p2 contain doclists. This function merges the content 4272 4555 ** of the two doclists together and sets buffer p1 to the result before 4273 4556 ** returning. 4274 4557 ** 4275 4558 ** If an error occurs, an error code is left in p->rc. If an error has ................................................................................ 4331 4614 }else{ 4332 4615 iNew = iPos2; 4333 4616 sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); 4334 4617 if( iPos1==iPos2 ){ 4335 4618 sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1); 4336 4619 } 4337 4620 } 4338 - p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); 4621 + if( iNew!=writer.iPrev || tmp.n==0 ){ 4622 + p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); 4623 + } 4339 4624 } 4340 4625 4341 4626 /* WRITEPOSLISTSIZE */ 4342 4627 fts5BufferSafeAppendVarint(&out, tmp.n * 2); 4343 4628 fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); 4344 4629 fts5DoclistIterNext(&i1); 4345 4630 fts5DoclistIterNext(&i2); ................................................................................ 4348 4633 4349 4634 fts5BufferSet(&p->rc, p1, out.n, out.p); 4350 4635 fts5BufferFree(&tmp); 4351 4636 fts5BufferFree(&out); 4352 4637 } 4353 4638 } 4354 4639 4355 -static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ 4356 - Fts5Buffer tmp = *p1; 4357 - *p1 = *p2; 4358 - *p2 = tmp; 4359 -} 4360 - 4361 4640 static void fts5SetupPrefixIter( 4362 4641 Fts5Index *p, /* Index to read from */ 4363 4642 int bDesc, /* True for "ORDER BY rowid DESC" */ 4364 4643 const u8 *pToken, /* Buffer containing prefix to match */ 4365 4644 int nToken, /* Size of buffer pToken in bytes */ 4366 4645 Fts5Colset *pColset, /* Restrict matches to these columns */ 4367 4646 Fts5IndexIter **ppIter /* OUT: New iterator */ 4368 4647 ){ 4369 4648 Fts5Structure *pStruct; 4370 4649 Fts5Buffer *aBuf; 4371 4650 const int nBuf = 32; 4651 + 4652 + void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); 4653 + int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*); 4654 + if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ 4655 + xMerge = fts5MergeRowidLists; 4656 + xAppend = fts5AppendRowid; 4657 + }else{ 4658 + xMerge = fts5MergePrefixLists; 4659 + xAppend = fts5AppendPoslist; 4660 + } 4372 4661 4373 4662 aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); 4374 4663 pStruct = fts5StructureRead(p); 4375 4664 4376 4665 if( aBuf && pStruct ){ 4377 4666 const int flags = FTS5INDEX_QUERY_SCAN; 4378 4667 int i; ................................................................................ 4398 4687 if( doclist.n>0 && iRowid<=iLastRowid ){ 4399 4688 for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ 4400 4689 assert( i<nBuf ); 4401 4690 if( aBuf[i].n==0 ){ 4402 4691 fts5BufferSwap(&doclist, &aBuf[i]); 4403 4692 fts5BufferZero(&doclist); 4404 4693 }else{ 4405 - fts5MergePrefixLists(p, &doclist, &aBuf[i]); 4694 + xMerge(p, &doclist, &aBuf[i]); 4406 4695 fts5BufferZero(&aBuf[i]); 4407 4696 } 4408 4697 } 4409 4698 iLastRowid = 0; 4410 4699 } 4411 4700 4412 - if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ 4701 + if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ 4413 4702 iLastRowid = iRowid; 4414 4703 } 4415 4704 } 4416 4705 4417 4706 for(i=0; i<nBuf; i++){ 4418 4707 if( p->rc==SQLITE_OK ){ 4419 - fts5MergePrefixLists(p, &doclist, &aBuf[i]); 4708 + xMerge(p, &doclist, &aBuf[i]); 4420 4709 } 4421 4710 fts5BufferFree(&aBuf[i]); 4422 4711 } 4423 4712 fts5MultiIterFree(p, p1); 4424 4713 4425 4714 pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); 4426 4715 if( pData ){ ................................................................................ 4442 4731 ** to the document with rowid iRowid. 4443 4732 */ 4444 4733 int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ 4445 4734 assert( p->rc==SQLITE_OK ); 4446 4735 4447 4736 /* Allocate the hash table if it has not already been allocated */ 4448 4737 if( p->pHash==0 ){ 4449 - p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); 4738 + p->rc = sqlite3Fts5HashNew(p->pConfig, &p->pHash, &p->nPendingData); 4450 4739 } 4451 4740 4452 4741 /* Flush the hash table to disk if required */ 4453 4742 if( iRowid<p->iWriteRowid 4454 4743 || (iRowid==p->iWriteRowid && p->bDelete==0) 4455 4744 || (p->nPendingData > p->pConfig->nHashSize) 4456 4745 ){ ................................................................................ 4563 4852 } 4564 4853 4565 4854 /* 4566 4855 ** Argument p points to a buffer containing utf-8 text that is n bytes in 4567 4856 ** size. Return the number of bytes in the nChar character prefix of the 4568 4857 ** buffer, or 0 if there are less than nChar characters in total. 4569 4858 */ 4570 -static int fts5IndexCharlenToBytelen(const char *p, int nByte, int nChar){ 4859 +int sqlite3Fts5IndexCharlenToBytelen( 4860 + const char *p, 4861 + int nByte, 4862 + int nChar 4863 +){ 4571 4864 int n = 0; 4572 4865 int i; 4573 4866 for(i=0; i<nChar; i++){ 4574 4867 if( n>=nByte ) return 0; /* Input contains fewer than nChar chars */ 4575 4868 if( (unsigned char)p[n++]>=0xc0 ){ 4576 4869 while( (p[n] & 0xc0)==0x80 ) n++; 4577 4870 } ................................................................................ 4620 4913 4621 4914 /* Add the entry to the main terms index. */ 4622 4915 rc = sqlite3Fts5HashWrite( 4623 4916 p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken 4624 4917 ); 4625 4918 4626 4919 for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ 4627 - int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); 4920 + const int nChar = pConfig->aPrefix[i]; 4921 + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); 4628 4922 if( nByte ){ 4629 4923 rc = sqlite3Fts5HashWrite(p->pHash, 4630 4924 p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, 4631 4925 nByte 4632 4926 ); 4633 4927 } 4634 4928 } ................................................................................ 4798 5092 Fts5IndexIter *pIter, 4799 5093 Fts5Colset *pColset, /* Column filter (or NULL) */ 4800 5094 const u8 **pp, /* OUT: Pointer to position-list data */ 4801 5095 int *pn, /* OUT: Size of position-list in bytes */ 4802 5096 i64 *piRowid /* OUT: Current rowid */ 4803 5097 ){ 4804 5098 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; 5099 + int eDetail = pIter->pIndex->pConfig->eDetail; 5100 + 4805 5101 assert( pIter->pIndex->rc==SQLITE_OK ); 4806 5102 *piRowid = pSeg->iRowid; 4807 - if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ 5103 + if( eDetail==FTS5_DETAIL_NONE ){ 5104 + *pn = pSeg->nPos; 5105 + }else 5106 + if( eDetail==FTS5_DETAIL_FULL 5107 + && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 5108 + ){ 4808 5109 u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; 4809 5110 if( pColset==0 || pIter->bFiltered ){ 4810 5111 *pn = pSeg->nPos; 4811 5112 *pp = pPos; 4812 5113 }else if( pColset->nCol==1 ){ 4813 5114 *pp = pPos; 4814 5115 *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); ................................................................................ 4817 5118 fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); 4818 5119 *pp = pIter->poslist.p; 4819 5120 *pn = pIter->poslist.n; 4820 5121 } 4821 5122 }else{ 4822 5123 fts5BufferZero(&pIter->poslist); 4823 5124 fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); 4824 - *pp = pIter->poslist.p; 5125 + if( eDetail==FTS5_DETAIL_FULL ){ 5126 + *pp = pIter->poslist.p; 5127 + } 4825 5128 *pn = pIter->poslist.n; 4826 5129 } 4827 5130 return fts5IndexReturn(pIter->pIndex); 4828 5131 } 5132 + 5133 +int sqlite3Fts5IterCollist( 5134 + Fts5IndexIter *pIter, 5135 + const u8 **pp, /* OUT: Pointer to position-list data */ 5136 + int *pn /* OUT: Size of position-list in bytes */ 5137 +){ 5138 + assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); 5139 + *pp = pIter->poslist.p; 5140 + *pn = pIter->poslist.n; 5141 + return SQLITE_OK; 5142 +} 4829 5143 4830 5144 /* 4831 5145 ** This function is similar to sqlite3Fts5IterPoslist(), except that it 4832 5146 ** copies the position list into the buffer supplied as the second 4833 5147 ** argument. 4834 5148 */ 4835 5149 int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ ................................................................................ 4936 5250 ** Below this point is the implementation of the integrity-check 4937 5251 ** functionality. 4938 5252 */ 4939 5253 4940 5254 /* 4941 5255 ** Return a simple checksum value based on the arguments. 4942 5256 */ 4943 -static u64 fts5IndexEntryCksum( 5257 +u64 sqlite3Fts5IndexEntryCksum( 4944 5258 i64 iRowid, 4945 5259 int iCol, 4946 5260 int iPos, 4947 5261 int iIdx, 4948 5262 const char *pTerm, 4949 5263 int nTerm 4950 5264 ){ ................................................................................ 5006 5320 Fts5Index *p, /* Fts5 index object */ 5007 5321 int iIdx, 5008 5322 const char *z, /* Index key to query for */ 5009 5323 int n, /* Size of index key in bytes */ 5010 5324 int flags, /* Flags for Fts5IndexQuery */ 5011 5325 u64 *pCksum /* IN/OUT: Checksum value */ 5012 5326 ){ 5327 + int eDetail = p->pConfig->eDetail; 5013 5328 u64 cksum = *pCksum; 5014 5329 Fts5IndexIter *pIdxIter = 0; 5330 + Fts5Buffer buf = {0, 0, 0}; 5015 5331 int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); 5016 5332 5017 5333 while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ 5018 - i64 dummy; 5019 - const u8 *pPos; 5020 - int nPos; 5021 5334 i64 rowid = sqlite3Fts5IterRowid(pIdxIter); 5022 - rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); 5335 + 5336 + if( eDetail==FTS5_DETAIL_NONE ){ 5337 + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n); 5338 + }else{ 5339 + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); 5340 + if( rc==SQLITE_OK ){ 5341 + Fts5PoslistReader sReader; 5342 + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); 5343 + sReader.bEof==0; 5344 + sqlite3Fts5PoslistReaderNext(&sReader) 5345 + ){ 5346 + int iCol = FTS5_POS2COLUMN(sReader.iPos); 5347 + int iOff = FTS5_POS2OFFSET(sReader.iPos); 5348 + cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); 5349 + } 5350 + } 5351 + } 5023 5352 if( rc==SQLITE_OK ){ 5024 - Fts5PoslistReader sReader; 5025 - for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); 5026 - sReader.bEof==0; 5027 - sqlite3Fts5PoslistReaderNext(&sReader) 5028 - ){ 5029 - int iCol = FTS5_POS2COLUMN(sReader.iPos); 5030 - int iOff = FTS5_POS2OFFSET(sReader.iPos); 5031 - cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); 5032 - } 5033 5353 rc = sqlite3Fts5IterNext(pIdxIter); 5034 5354 } 5035 5355 } 5036 5356 sqlite3Fts5IterClose(pIdxIter); 5357 + fts5BufferFree(&buf); 5037 5358 5038 5359 *pCksum = cksum; 5039 5360 return rc; 5040 5361 } 5041 5362 5042 5363 5043 5364 /* ................................................................................ 5323 5644 #endif 5324 5645 } 5325 5646 5326 5647 5327 5648 /* 5328 5649 ** Run internal checks to ensure that the FTS index (a) is internally 5329 5650 ** consistent and (b) contains entries for which the XOR of the checksums 5330 -** as calculated by fts5IndexEntryCksum() is cksum. 5651 +** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. 5331 5652 ** 5332 5653 ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the 5333 5654 ** checksum does not match. Return SQLITE_OK if all checks pass without 5334 5655 ** error, or some other SQLite error code if another error (e.g. OOM) 5335 5656 ** occurs. 5336 5657 */ 5337 5658 int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ 5659 + int eDetail = p->pConfig->eDetail; 5338 5660 u64 cksum2 = 0; /* Checksum based on contents of indexes */ 5339 5661 Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ 5340 5662 Fts5IndexIter *pIter; /* Used to iterate through entire index */ 5341 5663 Fts5Structure *pStruct; /* Index structure */ 5342 5664 5343 5665 #ifdef SQLITE_DEBUG 5344 5666 /* Used by extra internal tests only run if NDEBUG is not defined */ ................................................................................ 5382 5704 int iOff = 0; /* Offset within poslist */ 5383 5705 i64 iRowid = fts5MultiIterRowid(pIter); 5384 5706 char *z = (char*)fts5MultiIterTerm(pIter, &n); 5385 5707 5386 5708 /* If this is a new term, query for it. Update cksum3 with the results. */ 5387 5709 fts5TestTerm(p, &term, z, n, cksum2, &cksum3); 5388 5710 5389 - poslist.n = 0; 5390 - fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist); 5391 - while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ 5392 - int iCol = FTS5_POS2COLUMN(iPos); 5393 - int iTokOff = FTS5_POS2OFFSET(iPos); 5394 - cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); 5711 + if( eDetail==FTS5_DETAIL_NONE ){ 5712 + if( 0==fts5MultiIterIsEmpty(p, pIter) ){ 5713 + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, 0, 0, -1, z, n); 5714 + } 5715 + }else{ 5716 + poslist.n = 0; 5717 + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); 5718 + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ 5719 + int iCol = FTS5_POS2COLUMN(iPos); 5720 + int iTokOff = FTS5_POS2OFFSET(iPos); 5721 + cksum2 ^= sqlite3Fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); 5722 + } 5395 5723 } 5396 5724 } 5397 5725 fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); 5398 5726 5399 5727 fts5MultiIterFree(p, pIter); 5400 5728 if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; 5401 5729 ................................................................................ 5403 5731 #ifdef SQLITE_DEBUG 5404 5732 fts5BufferFree(&term); 5405 5733 #endif 5406 5734 fts5BufferFree(&poslist); 5407 5735 return fts5IndexReturn(p); 5408 5736 } 5409 5737 5410 - 5411 -/* 5412 -** Calculate and return a checksum that is the XOR of the index entry 5413 -** checksum of all entries that would be generated by the token specified 5414 -** by the final 5 arguments. 5415 -*/ 5416 -u64 sqlite3Fts5IndexCksum( 5417 - Fts5Config *pConfig, /* Configuration object */ 5418 - i64 iRowid, /* Document term appears in */ 5419 - int iCol, /* Column term appears in */ 5420 - int iPos, /* Position term appears in */ 5421 - const char *pTerm, int nTerm /* Term at iPos */ 5422 -){ 5423 - u64 ret = 0; /* Return value */ 5424 - int iIdx; /* For iterating through indexes */ 5425 - 5426 - ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); 5427 - 5428 - for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){ 5429 - int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); 5430 - if( nByte ){ 5431 - ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); 5432 - } 5433 - } 5434 - 5435 - return ret; 5436 -} 5437 - 5438 5738 /************************************************************************* 5439 5739 ************************************************************************** 5440 5740 ** Below this point is the implementation of the fts5_decode() scalar 5441 5741 ** function only. 5442 5742 */ 5443 5743 5444 5744 /* ................................................................................ 5788 6088 if( rc==SQLITE_OK ){ 5789 6089 rc = sqlite3_create_function( 5790 6090 db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 5791 6091 ); 5792 6092 } 5793 6093 return rc; 5794 6094 } 5795 -
Changes to ext/fts5/fts5_main.c.
222 222 */ 223 223 #define FTS5CSR_REQUIRE_CONTENT 0x01 224 224 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 225 225 #define FTS5CSR_REQUIRE_INST 0x04 226 226 #define FTS5CSR_EOF 0x08 227 227 #define FTS5CSR_FREE_ZRANK 0x10 228 228 #define FTS5CSR_REQUIRE_RESEEK 0x20 229 +#define FTS5CSR_REQUIRE_POSLIST 0x40 229 230 230 231 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) 231 232 #define BitFlagTest(x,y) (((x) & (y))!=0) 232 233 233 234 234 235 /* 235 236 ** Macros to Set(), Clear() and Test() cursor flags. ................................................................................ 635 636 ** specific to the previous row stored by the cursor object. 636 637 */ 637 638 static void fts5CsrNewrow(Fts5Cursor *pCsr){ 638 639 CsrFlagSet(pCsr, 639 640 FTS5CSR_REQUIRE_CONTENT 640 641 | FTS5CSR_REQUIRE_DOCSIZE 641 642 | FTS5CSR_REQUIRE_INST 643 + | FTS5CSR_REQUIRE_POSLIST 642 644 ); 643 645 } 644 646 645 647 static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ 646 648 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); 647 649 Fts5Auxdata *pData; 648 650 Fts5Auxdata *pNext; ................................................................................ 717 719 int iOff = 0; 718 720 rc = SQLITE_OK; 719 721 720 722 pSorter->iRowid = sqlite3_column_int64(pSorter->pStmt, 0); 721 723 nBlob = sqlite3_column_bytes(pSorter->pStmt, 1); 722 724 aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1); 723 725 724 - for(i=0; i<(pSorter->nIdx-1); i++){ 725 - int iVal; 726 - a += fts5GetVarint32(a, iVal); 727 - iOff += iVal; 728 - pSorter->aIdx[i] = iOff; 726 + /* nBlob==0 in detail=none mode. */ 727 + if( nBlob>0 ){ 728 + for(i=0; i<(pSorter->nIdx-1); i++){ 729 + int iVal; 730 + a += fts5GetVarint32(a, iVal); 731 + iOff += iVal; 732 + pSorter->aIdx[i] = iOff; 733 + } 734 + pSorter->aIdx[i] = &aBlob[nBlob] - a; 735 + pSorter->aPoslist = a; 729 736 } 730 - pSorter->aIdx[i] = &aBlob[nBlob] - a; 731 737 732 - pSorter->aPoslist = a; 733 738 fts5CsrNewrow(pCsr); 734 739 } 735 740 736 741 return rc; 737 742 } 738 743 739 744 ................................................................................ 1163 1168 assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); 1164 1169 assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); 1165 1170 assert( pCsr->iLastRowid==LARGEST_INT64 ); 1166 1171 assert( pCsr->iFirstRowid==SMALLEST_INT64 ); 1167 1172 pCsr->ePlan = FTS5_PLAN_SOURCE; 1168 1173 pCsr->pExpr = pTab->pSortCsr->pExpr; 1169 1174 rc = fts5CursorFirst(pTab, pCsr, bDesc); 1175 + sqlite3Fts5ExprClearEof(pCsr->pExpr); 1170 1176 }else if( pMatch ){ 1171 1177 const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 1172 1178 if( zExpr==0 ) zExpr = ""; 1173 1179 1174 1180 rc = fts5CursorParseRank(pConfig, pCsr, pRank); 1175 1181 if( rc==SQLITE_OK ){ 1176 1182 if( zExpr[0]=='*' ){ ................................................................................ 1592 1598 int rc; 1593 1599 Fts5Table *pTab = (Fts5Table*)pVtab; 1594 1600 fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); 1595 1601 rc = sqlite3Fts5StorageRollback(pTab->pStorage); 1596 1602 return rc; 1597 1603 } 1598 1604 1605 +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*); 1606 + 1599 1607 static void *fts5ApiUserData(Fts5Context *pCtx){ 1600 1608 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1601 1609 return pCsr->pAux->pUserData; 1602 1610 } 1603 1611 1604 1612 static int fts5ApiColumnCount(Fts5Context *pCtx){ 1605 1613 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; ................................................................................ 1641 1649 } 1642 1650 1643 1651 static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ 1644 1652 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1645 1653 return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); 1646 1654 } 1647 1655 1648 -static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ 1649 - int n; 1650 - if( pCsr->pSorter ){ 1656 +static int fts5ApiColumnText( 1657 + Fts5Context *pCtx, 1658 + int iCol, 1659 + const char **pz, 1660 + int *pn 1661 +){ 1662 + int rc = SQLITE_OK; 1663 + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1664 + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ 1665 + *pz = 0; 1666 + *pn = 0; 1667 + }else{ 1668 + rc = fts5SeekCursor(pCsr, 0); 1669 + if( rc==SQLITE_OK ){ 1670 + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1671 + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1672 + } 1673 + } 1674 + return rc; 1675 +} 1676 + 1677 +static int fts5CsrPoslist( 1678 + Fts5Cursor *pCsr, 1679 + int iPhrase, 1680 + const u8 **pa, 1681 + int *pn 1682 +){ 1683 + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; 1684 + int rc = SQLITE_OK; 1685 + int bLive = (pCsr->pSorter==0); 1686 + 1687 + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ 1688 + 1689 + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ 1690 + Fts5PoslistPopulator *aPopulator; 1691 + int i; 1692 + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); 1693 + if( aPopulator==0 ) rc = SQLITE_NOMEM; 1694 + for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ 1695 + int n; const char *z; 1696 + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); 1697 + if( rc==SQLITE_OK ){ 1698 + rc = sqlite3Fts5ExprPopulatePoslists( 1699 + pConfig, pCsr->pExpr, aPopulator, i, z, n 1700 + ); 1701 + } 1702 + } 1703 + sqlite3_free(aPopulator); 1704 + 1705 + if( pCsr->pSorter ){ 1706 + sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid); 1707 + } 1708 + } 1709 + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); 1710 + } 1711 + 1712 + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ 1651 1713 Fts5Sorter *pSorter = pCsr->pSorter; 1652 1714 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); 1653 - n = pSorter->aIdx[iPhrase] - i1; 1715 + *pn = pSorter->aIdx[iPhrase] - i1; 1654 1716 *pa = &pSorter->aPoslist[i1]; 1655 1717 }else{ 1656 - n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); 1718 + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); 1657 1719 } 1658 - return n; 1720 + 1721 + return rc; 1659 1722 } 1660 1723 1661 1724 /* 1662 1725 ** Ensure that the Fts5Cursor.nInstCount and aInst[] variables are populated 1663 1726 ** correctly for the current view. Return SQLITE_OK if successful, or an 1664 1727 ** SQLite error code otherwise. 1665 1728 */ ................................................................................ 1676 1739 aIter = pCsr->aInstIter; 1677 1740 1678 1741 if( aIter ){ 1679 1742 int nInst = 0; /* Number instances seen so far */ 1680 1743 int i; 1681 1744 1682 1745 /* Initialize all iterators */ 1683 - for(i=0; i<nIter; i++){ 1746 + for(i=0; i<nIter && rc==SQLITE_OK; i++){ 1684 1747 const u8 *a; 1685 - int n = fts5CsrPoslist(pCsr, i, &a); 1748 + int n; 1749 + rc = fts5CsrPoslist(pCsr, i, &a, &n); 1686 1750 sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); 1687 1751 } 1688 1752 1689 - while( 1 ){ 1690 - int *aInst; 1691 - int iBest = -1; 1692 - for(i=0; i<nIter; i++){ 1693 - if( (aIter[i].bEof==0) 1694 - && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 1695 - ){ 1696 - iBest = i; 1753 + if( rc==SQLITE_OK ){ 1754 + while( 1 ){ 1755 + int *aInst; 1756 + int iBest = -1; 1757 + for(i=0; i<nIter; i++){ 1758 + if( (aIter[i].bEof==0) 1759 + && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 1760 + ){ 1761 + iBest = i; 1762 + } 1697 1763 } 1698 - } 1699 - if( iBest<0 ) break; 1764 + if( iBest<0 ) break; 1700 1765 1701 - nInst++; 1702 - if( nInst>=pCsr->nInstAlloc ){ 1703 - pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; 1704 - aInst = (int*)sqlite3_realloc( 1705 - pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 1706 - ); 1707 - if( aInst ){ 1708 - pCsr->aInst = aInst; 1709 - }else{ 1710 - rc = SQLITE_NOMEM; 1711 - break; 1766 + nInst++; 1767 + if( nInst>=pCsr->nInstAlloc ){ 1768 + pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; 1769 + aInst = (int*)sqlite3_realloc( 1770 + pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 1771 + ); 1772 + if( aInst ){ 1773 + pCsr->aInst = aInst; 1774 + }else{ 1775 + rc = SQLITE_NOMEM; 1776 + break; 1777 + } 1712 1778 } 1779 + 1780 + aInst = &pCsr->aInst[3 * (nInst-1)]; 1781 + aInst[0] = iBest; 1782 + aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); 1783 + aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); 1784 + sqlite3Fts5PoslistReaderNext(&aIter[iBest]); 1713 1785 } 1714 - 1715 - aInst = &pCsr->aInst[3 * (nInst-1)]; 1716 - aInst[0] = iBest; 1717 - aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); 1718 - aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); 1719 - sqlite3Fts5PoslistReaderNext(&aIter[iBest]); 1720 1786 } 1721 1787 1722 1788 pCsr->nInstCount = nInst; 1723 1789 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_INST); 1724 1790 } 1725 1791 return rc; 1726 1792 } ................................................................................ 1745 1811 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1746 1812 int rc = SQLITE_OK; 1747 1813 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 1748 1814 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 1749 1815 ){ 1750 1816 if( iIdx<0 || iIdx>=pCsr->nInstCount ){ 1751 1817 rc = SQLITE_RANGE; 1818 +#if 0 1819 + }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ 1820 + *piPhrase = pCsr->aInst[iIdx*3]; 1821 + *piCol = pCsr->aInst[iIdx*3 + 2]; 1822 + *piOff = -1; 1823 +#endif 1752 1824 }else{ 1753 1825 *piPhrase = pCsr->aInst[iIdx*3]; 1754 1826 *piCol = pCsr->aInst[iIdx*3 + 1]; 1755 1827 *piOff = pCsr->aInst[iIdx*3 + 2]; 1756 1828 } 1757 1829 } 1758 1830 return rc; 1759 1831 } 1760 1832 1761 1833 static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ 1762 1834 return fts5CursorRowid((Fts5Cursor*)pCtx); 1763 1835 } 1764 1836 1765 -static int fts5ApiColumnText( 1766 - Fts5Context *pCtx, 1767 - int iCol, 1768 - const char **pz, 1769 - int *pn 1770 -){ 1771 - int rc = SQLITE_OK; 1772 - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1773 - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ 1774 - *pz = 0; 1775 - *pn = 0; 1776 - }else{ 1777 - rc = fts5SeekCursor(pCsr, 0); 1778 - if( rc==SQLITE_OK ){ 1779 - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1780 - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1781 - } 1782 - } 1783 - return rc; 1784 -} 1785 - 1786 1837 static int fts5ColumnSizeCb( 1787 1838 void *pContext, /* Pointer to int */ 1788 1839 int tflags, 1789 1840 const char *pToken, /* Buffer containing token */ 1790 1841 int nToken, /* Size of token in bytes */ 1791 1842 int iStart, /* Start offset of token */ 1792 1843 int iEnd /* End offset of token */ ................................................................................ 1923 1974 *piOff = 0; 1924 1975 pIter->a += fts5GetVarint32(pIter->a, iVal); 1925 1976 } 1926 1977 *piOff += (iVal-2); 1927 1978 } 1928 1979 } 1929 1980 1930 -static void fts5ApiPhraseFirst( 1981 +static int fts5ApiPhraseFirst( 1931 1982 Fts5Context *pCtx, 1932 1983 int iPhrase, 1933 1984 Fts5PhraseIter *pIter, 1934 1985 int *piCol, int *piOff 1935 1986 ){ 1936 1987 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1937 - int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a); 1938 - pIter->b = &pIter->a[n]; 1939 - *piCol = 0; 1940 - *piOff = 0; 1941 - fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); 1942 -} 1988 + int n; 1989 + int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); 1990 + if( rc==SQLITE_OK ){ 1991 + pIter->b = &pIter->a[n]; 1992 + *piCol = 0; 1993 + *piOff = 0; 1994 + fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); 1995 + } 1996 + return rc; 1997 +} 1998 + 1999 +static void fts5ApiPhraseNextColumn( 2000 + Fts5Context *pCtx, 2001 + Fts5PhraseIter *pIter, 2002 + int *piCol 2003 +){ 2004 + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 2005 + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; 2006 + 2007 + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ 2008 + if( pIter->a>=pIter->b ){ 2009 + *piCol = -1; 2010 + }else{ 2011 + int iIncr; 2012 + pIter->a += fts5GetVarint32(&pIter->a[0], iIncr); 2013 + *piCol += (iIncr-2); 2014 + } 2015 + }else{ 2016 + while( 1 ){ 2017 + int dummy; 2018 + if( pIter->a>=pIter->b ){ 2019 + *piCol = -1; 2020 + return; 2021 + } 2022 + if( pIter->a[0]==0x01 ) break; 2023 + pIter->a += fts5GetVarint32(pIter->a, dummy); 2024 + } 2025 + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); 2026 + } 2027 +} 2028 + 2029 +static int fts5ApiPhraseFirstColumn( 2030 + Fts5Context *pCtx, 2031 + int iPhrase, 2032 + Fts5PhraseIter *pIter, 2033 + int *piCol 2034 +){ 2035 + int rc = SQLITE_OK; 2036 + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 2037 + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; 2038 + 2039 + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ 2040 + int n; 2041 + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); 2042 + if( rc==SQLITE_OK ){ 2043 + pIter->b = &pIter->a[n]; 2044 + *piCol = 0; 2045 + fts5ApiPhraseNextColumn(pCtx, pIter, piCol); 2046 + } 2047 + }else{ 2048 + int n; 2049 + rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); 2050 + if( rc==SQLITE_OK ){ 2051 + pIter->b = &pIter->a[n]; 2052 + if( n<=0 ){ 2053 + *piCol = -1; 2054 + }else if( pIter->a[0]==0x01 ){ 2055 + pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol); 2056 + }else{ 2057 + *piCol = 0; 2058 + } 2059 + } 2060 + } 2061 + 2062 + return rc; 2063 +} 2064 + 1943 2065 1944 2066 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 1945 2067 int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) 1946 2068 ); 1947 2069 1948 2070 static const Fts5ExtensionApi sFts5Api = { 1949 2071 2, /* iVersion */ ................................................................................ 1960 2082 fts5ApiColumnText, 1961 2083 fts5ApiColumnSize, 1962 2084 fts5ApiQueryPhrase, 1963 2085 fts5ApiSetAuxdata, 1964 2086 fts5ApiGetAuxdata, 1965 2087 fts5ApiPhraseFirst, 1966 2088 fts5ApiPhraseNext, 2089 + fts5ApiPhraseFirstColumn, 2090 + fts5ApiPhraseNextColumn, 1967 2091 }; 1968 - 1969 2092 1970 2093 /* 1971 2094 ** Implementation of API function xQueryPhrase(). 1972 2095 */ 1973 2096 static int fts5ApiQueryPhrase( 1974 2097 Fts5Context *pCtx, 1975 2098 int iPhrase, ................................................................................ 2094 2217 static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ 2095 2218 int i; 2096 2219 int rc = SQLITE_OK; 2097 2220 int nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); 2098 2221 Fts5Buffer val; 2099 2222 2100 2223 memset(&val, 0, sizeof(Fts5Buffer)); 2101 - 2102 - /* Append the varints */ 2103 - for(i=0; i<(nPhrase-1); i++){ 2104 - const u8 *dummy; 2105 - int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); 2106 - sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); 2107 - } 2108 - 2109 - /* Append the position lists */ 2110 - for(i=0; i<nPhrase; i++){ 2111 - const u8 *pPoslist; 2112 - int nPoslist; 2113 - nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); 2114 - sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); 2224 + switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){ 2225 + case FTS5_DETAIL_FULL: 2226 + 2227 + /* Append the varints */ 2228 + for(i=0; i<(nPhrase-1); i++){ 2229 + const u8 *dummy; 2230 + int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy); 2231 + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); 2232 + } 2233 + 2234 + /* Append the position lists */ 2235 + for(i=0; i<nPhrase; i++){ 2236 + const u8 *pPoslist; 2237 + int nPoslist; 2238 + nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist); 2239 + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); 2240 + } 2241 + break; 2242 + 2243 + case FTS5_DETAIL_COLUMNS: 2244 + 2245 + /* Append the varints */ 2246 + for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){ 2247 + const u8 *dummy; 2248 + int nByte; 2249 + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte); 2250 + sqlite3Fts5BufferAppendVarint(&rc, &val, nByte); 2251 + } 2252 + 2253 + /* Append the position lists */ 2254 + for(i=0; rc==SQLITE_OK && i<nPhrase; i++){ 2255 + const u8 *pPoslist; 2256 + int nPoslist; 2257 + rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist); 2258 + sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist); 2259 + } 2260 + break; 2261 + 2262 + default: 2263 + break; 2115 2264 } 2116 2265 2117 2266 sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free); 2118 2267 return rc; 2119 2268 } 2120 2269 2121 2270 /*
Changes to ext/fts5/fts5_storage.c.
821 821 */ 822 822 typedef struct Fts5IntegrityCtx Fts5IntegrityCtx; 823 823 struct Fts5IntegrityCtx { 824 824 i64 iRowid; 825 825 int iCol; 826 826 int szCol; 827 827 u64 cksum; 828 + Fts5Termset *pTermset; 828 829 Fts5Config *pConfig; 829 830 }; 831 + 830 832 831 833 /* 832 834 ** Tokenization callback used by integrity check. 833 835 */ 834 836 static int fts5StorageIntegrityCallback( 835 - void *pContext, /* Pointer to Fts5InsertCtx object */ 837 + void *pContext, /* Pointer to Fts5IntegrityCtx object */ 836 838 int tflags, 837 839 const char *pToken, /* Buffer containing token */ 838 840 int nToken, /* Size of token in bytes */ 839 841 int iStart, /* Start offset of token */ 840 842 int iEnd /* End offset of token */ 841 843 ){ 842 844 Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext; 845 + Fts5Termset *pTermset = pCtx->pTermset; 846 + int bPresent; 847 + int ii; 848 + int rc = SQLITE_OK; 849 + int iPos; 850 + int iCol; 851 + 843 852 if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){ 844 853 pCtx->szCol++; 845 854 } 846 - pCtx->cksum ^= sqlite3Fts5IndexCksum( 847 - pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken 848 - ); 849 - return SQLITE_OK; 855 + 856 + switch( pCtx->pConfig->eDetail ){ 857 + case FTS5_DETAIL_FULL: 858 + iPos = pCtx->szCol-1; 859 + iCol = pCtx->iCol; 860 + break; 861 + 862 + case FTS5_DETAIL_COLUMNS: 863 + iPos = pCtx->iCol; 864 + iCol = 0; 865 + break; 866 + 867 + default: 868 + assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE ); 869 + iPos = 0; 870 + iCol = 0; 871 + break; 872 + } 873 + 874 + rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent); 875 + if( rc==SQLITE_OK && bPresent==0 ){ 876 + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( 877 + pCtx->iRowid, iCol, iPos, 0, pToken, nToken 878 + ); 879 + } 880 + 881 + for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){ 882 + const int nChar = pCtx->pConfig->aPrefix[ii]; 883 + int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); 884 + if( nByte ){ 885 + rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent); 886 + if( bPresent==0 ){ 887 + pCtx->cksum ^= sqlite3Fts5IndexEntryCksum( 888 + pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte 889 + ); 890 + } 891 + } 892 + } 893 + 894 + return rc; 850 895 } 851 896 852 897 /* 853 898 ** Check that the contents of the FTS index match that of the %_content 854 899 ** table. Return SQLITE_OK if they do, or SQLITE_CORRUPT if not. Return 855 900 ** some other SQLite error code if an error occurs while attempting to 856 901 ** determine this. ................................................................................ 877 922 int rc2; 878 923 while( SQLITE_ROW==sqlite3_step(pScan) ){ 879 924 int i; 880 925 ctx.iRowid = sqlite3_column_int64(pScan, 0); 881 926 ctx.szCol = 0; 882 927 if( pConfig->bColumnsize ){ 883 928 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); 929 + } 930 + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ 931 + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); 884 932 } 885 933 for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ 886 934 if( pConfig->abUnindexed[i] ) continue; 887 935 ctx.iCol = i; 888 936 ctx.szCol = 0; 889 - rc = sqlite3Fts5Tokenize(pConfig, 890 - FTS5_TOKENIZE_DOCUMENT, 891 - (const char*)sqlite3_column_text(pScan, i+1), 892 - sqlite3_column_bytes(pScan, i+1), 893 - (void*)&ctx, 894 - fts5StorageIntegrityCallback 895 - ); 896 - if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ 937 + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ 938 + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); 939 + } 940 + if( rc==SQLITE_OK ){ 941 + rc = sqlite3Fts5Tokenize(pConfig, 942 + FTS5_TOKENIZE_DOCUMENT, 943 + (const char*)sqlite3_column_text(pScan, i+1), 944 + sqlite3_column_bytes(pScan, i+1), 945 + (void*)&ctx, 946 + fts5StorageIntegrityCallback 947 + ); 948 + } 949 + if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ 897 950 rc = FTS5_CORRUPT; 898 951 } 899 952 aTotalSize[i] += ctx.szCol; 953 + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ 954 + sqlite3Fts5TermsetFree(ctx.pTermset); 955 + ctx.pTermset = 0; 956 + } 900 957 } 958 + sqlite3Fts5TermsetFree(ctx.pTermset); 959 + ctx.pTermset = 0; 960 + 901 961 if( rc!=SQLITE_OK ) break; 902 962 } 903 963 rc2 = sqlite3_reset(pScan); 904 964 if( rc==SQLITE_OK ) rc = rc2; 905 965 } 906 966 907 967 /* Test that the "totals" (sometimes called "averages") record looks Ok */
Changes to ext/fts5/fts5_tcl.c.
231 231 { "xColumnText", 1, "COL" }, /* 9 */ 232 232 { "xColumnSize", 1, "COL" }, /* 10 */ 233 233 { "xQueryPhrase", 2, "PHRASE SCRIPT" }, /* 11 */ 234 234 { "xSetAuxdata", 1, "VALUE" }, /* 12 */ 235 235 { "xGetAuxdata", 1, "CLEAR" }, /* 13 */ 236 236 { "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */ 237 237 { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ 238 + { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ 239 + { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ 238 240 { 0, 0, 0} 239 241 }; 240 242 241 243 int rc; 242 244 int iSub = 0; 243 245 F5tApi *p = (F5tApi*)clientData; 244 246 ................................................................................ 424 426 } 425 427 CASE(15, "xGetAuxdataInt") { 426 428 int iVal; 427 429 int bClear; 428 430 if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR; 429 431 iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0); 430 432 Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal)); 433 + break; 434 + } 435 + 436 + CASE(16, "xPhraseForeach") { 437 + int iPhrase; 438 + int iCol; 439 + int iOff; 440 + const char *zColvar; 441 + const char *zOffvar; 442 + Tcl_Obj *pScript = objv[5]; 443 + Fts5PhraseIter iter; 444 + 445 + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; 446 + zColvar = Tcl_GetString(objv[3]); 447 + zOffvar = Tcl_GetString(objv[4]); 448 + 449 + for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff); 450 + iCol>=0; 451 + p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) 452 + ){ 453 + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); 454 + Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0); 455 + rc = Tcl_EvalObjEx(interp, pScript, 0); 456 + if( rc==TCL_CONTINUE ) rc = TCL_OK; 457 + if( rc!=TCL_OK ){ 458 + if( rc==TCL_BREAK ) rc = TCL_OK; 459 + break; 460 + } 461 + } 462 + 463 + break; 464 + } 465 + 466 + CASE(17, "xPhraseColumnForeach") { 467 + int iPhrase; 468 + int iCol; 469 + const char *zColvar; 470 + Tcl_Obj *pScript = objv[4]; 471 + Fts5PhraseIter iter; 472 + 473 + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; 474 + zColvar = Tcl_GetString(objv[3]); 475 + 476 + for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol); 477 + iCol>=0; 478 + p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol) 479 + ){ 480 + Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0); 481 + rc = Tcl_EvalObjEx(interp, pScript, 0); 482 + if( rc==TCL_CONTINUE ) rc = TCL_OK; 483 + if( rc!=TCL_OK ){ 484 + if( rc==TCL_BREAK ) rc = TCL_OK; 485 + break; 486 + } 487 + } 488 + 431 489 break; 432 490 } 433 491 434 492 default: 435 493 assert( 0 ); 436 494 break; 437 495 }
Changes to ext/fts5/fts5_test_mi.c.
130 130 ){ 131 131 Fts5PhraseIter iter; 132 132 int iCol, iOff; 133 133 u32 *aOut = (u32*)pUserData; 134 134 int iPrev = -1; 135 135 136 136 for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff); 137 - iOff>=0; 137 + iCol>=0; 138 138 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) 139 139 ){ 140 140 aOut[iCol*3+1]++; 141 141 if( iCol!=iPrev ) aOut[iCol*3 + 2]++; 142 142 iPrev = iCol; 143 143 } 144 144 ................................................................................ 207 207 char f, 208 208 u32 *aOut 209 209 ){ 210 210 int i; 211 211 int rc = SQLITE_OK; 212 212 213 213 switch( f ){ 214 - case 'b': 214 + case 'b': { 215 + int iPhrase; 216 + int nInt = ((p->nCol + 31) / 32) * p->nPhrase; 217 + for(i=0; i<nInt; i++) aOut[i] = 0; 218 + 219 + for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ 220 + Fts5PhraseIter iter; 221 + int iCol; 222 + for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); 223 + iCol>=0; 224 + pApi->xPhraseNextColumn(pFts, &iter, &iCol) 225 + ){ 226 + aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32); 227 + } 228 + } 229 + 230 + break; 231 + } 232 + 215 233 case 'x': 216 234 case 'y': { 217 235 int nMul = (f=='x' ? 3 : 1); 218 236 int iPhrase; 219 237 220 - if( f=='b' ){ 221 - int nInt = ((p->nCol + 31) / 32) * p->nPhrase; 222 - for(i=0; i<nInt; i++) aOut[i] = 0; 223 - }else{ 224 - for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; 225 - } 238 + for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0; 226 239 227 240 for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){ 228 241 Fts5PhraseIter iter; 229 242 int iOff, iCol; 230 243 for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); 231 244 iOff>=0; 232 245 pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
Changes to ext/fts5/fts5_vocab.c.
375 375 int rc = SQLITE_OK; 376 376 int nCol = pCsr->pConfig->nCol; 377 377 378 378 pCsr->rowid++; 379 379 380 380 if( pTab->eType==FTS5_VOCAB_COL ){ 381 381 for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){ 382 - if( pCsr->aCnt[pCsr->iCol] ) break; 382 + if( pCsr->aDoc[pCsr->iCol] ) break; 383 383 } 384 384 } 385 385 386 386 if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ 387 387 if( sqlite3Fts5IterEof(pCsr->pIter) ){ 388 388 pCsr->bEof = 1; 389 389 }else{ ................................................................................ 408 408 assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); 409 409 while( rc==SQLITE_OK ){ 410 410 i64 dummy; 411 411 const u8 *pPos; int nPos; /* Position list */ 412 412 i64 iPos = 0; /* 64-bit position read from poslist */ 413 413 int iOff = 0; /* Current offset within position list */ 414 414 415 - rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); 416 - if( rc==SQLITE_OK ){ 417 - if( pTab->eType==FTS5_VOCAB_ROW ){ 418 - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ 419 - pCsr->aCnt[0]++; 415 + switch( pCsr->pConfig->eDetail ){ 416 + case FTS5_DETAIL_FULL: 417 + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); 418 + if( rc==SQLITE_OK ){ 419 + if( pTab->eType==FTS5_VOCAB_ROW ){ 420 + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ 421 + pCsr->aCnt[0]++; 422 + } 423 + pCsr->aDoc[0]++; 424 + }else{ 425 + int iCol = -1; 426 + while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ 427 + int ii = FTS5_POS2COLUMN(iPos); 428 + pCsr->aCnt[ii]++; 429 + if( iCol!=ii ){ 430 + pCsr->aDoc[ii]++; 431 + iCol = ii; 432 + } 433 + } 434 + } 420 435 } 421 - pCsr->aDoc[0]++; 422 - }else{ 423 - int iCol = -1; 424 - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ 425 - int ii = FTS5_POS2COLUMN(iPos); 426 - pCsr->aCnt[ii]++; 427 - if( iCol!=ii ){ 428 - pCsr->aDoc[ii]++; 429 - iCol = ii; 436 + break; 437 + 438 + case FTS5_DETAIL_COLUMNS: 439 + if( pTab->eType==FTS5_VOCAB_ROW ){ 440 + pCsr->aDoc[0]++; 441 + }else{ 442 + Fts5Buffer buf = {0, 0, 0}; 443 + rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf); 444 + if( rc==SQLITE_OK ){ 445 + while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){ 446 + assert_nc( iPos>=0 && iPos<nCol ); 447 + if( iPos<nCol ) pCsr->aDoc[iPos]++; 448 + } 430 449 } 450 + sqlite3Fts5BufferFree(&buf); 431 451 } 432 - } 452 + break; 453 + 454 + default: 455 + assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); 456 + pCsr->aDoc[0]++; 457 + break; 458 + } 459 + 460 + if( rc==SQLITE_OK ){ 433 461 rc = sqlite3Fts5IterNextScan(pCsr->pIter); 434 462 } 435 463 436 464 if( rc==SQLITE_OK ){ 437 465 zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); 438 466 if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ 439 467 break; ................................................................................ 441 469 if( sqlite3Fts5IterEof(pCsr->pIter) ) break; 442 470 } 443 471 } 444 472 } 445 473 } 446 474 447 475 if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ 448 - while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++; 476 + while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; 449 477 assert( pCsr->iCol<pCsr->pConfig->nCol ); 450 478 } 451 479 return rc; 452 480 } 453 481 454 482 /* 455 483 ** This is the xFilter implementation for the virtual table. ................................................................................ 521 549 522 550 static int fts5VocabColumnMethod( 523 551 sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ 524 552 sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ 525 553 int iCol /* Index of column to read value from */ 526 554 ){ 527 555 Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; 556 + int eDetail = pCsr->pConfig->eDetail; 557 + int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; 558 + i64 iVal = 0; 528 559 529 560 if( iCol==0 ){ 530 561 sqlite3_result_text( 531 562 pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT 532 563 ); 533 - } 534 - else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){ 564 + }else if( eType==FTS5_VOCAB_COL ){ 535 565 assert( iCol==1 || iCol==2 || iCol==3 ); 536 566 if( iCol==1 ){ 537 - const char *z = pCsr->pConfig->azCol[pCsr->iCol]; 538 - sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); 567 + if( eDetail!=FTS5_DETAIL_NONE ){ 568 + const char *z = pCsr->pConfig->azCol[pCsr->iCol]; 569 + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); 570 + } 539 571 }else if( iCol==2 ){ 540 - sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]); 572 + iVal = pCsr->aDoc[pCsr->iCol]; 541 573 }else{ 542 - sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]); 574 + iVal = pCsr->aCnt[pCsr->iCol]; 543 575 } 544 576 }else{ 545 577 assert( iCol==1 || iCol==2 ); 546 578 if( iCol==1 ){ 547 - sqlite3_result_int64(pCtx, pCsr->aDoc[0]); 579 + iVal = pCsr->aDoc[0]; 548 580 }else{ 549 - sqlite3_result_int64(pCtx, pCsr->aCnt[0]); 581 + iVal = pCsr->aCnt[0]; 550 582 } 551 583 } 584 + 585 + if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); 552 586 return SQLITE_OK; 553 587 } 554 588 555 589 /* 556 590 ** This is the xRowid method. The SQLite core calls this routine to 557 591 ** retrieve the rowid for the current row of the result set. The 558 592 ** rowid should be written to *pRowid.
Changes to ext/fts5/test/fts5_common.tcl.
23 23 proc fts5_test_poslist {cmd} { 24 24 set res [list] 25 25 for {set i 0} {$i < [$cmd xInstCount]} {incr i} { 26 26 lappend res [string map {{ } .} [$cmd xInst $i]] 27 27 } 28 28 set res 29 29 } 30 + 31 +proc fts5_test_poslist2 {cmd} { 32 + set res [list] 33 + 34 + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { 35 + $cmd xPhraseForeach $i c o { 36 + lappend res $i.$c.$o 37 + } 38 + } 39 + 40 + set res 41 +} 42 + 43 +proc fts5_test_collist {cmd} { 44 + set res [list] 45 + 46 + for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} { 47 + $cmd xPhraseColumnForeach $i c { lappend res $i.$c } 48 + } 49 + 50 + set res 51 +} 30 52 31 53 proc fts5_test_columnsize {cmd} { 32 54 set res [list] 33 55 for {set i 0} {$i < [$cmd xColumnCount]} {incr i} { 34 56 lappend res [$cmd xColumnSize $i] 35 57 } 36 58 set res ................................................................................ 109 131 110 132 proc fts5_aux_test_functions {db} { 111 133 foreach f { 112 134 fts5_test_columnsize 113 135 fts5_test_columntext 114 136 fts5_test_columntotalsize 115 137 fts5_test_poslist 138 + fts5_test_poslist2 139 + fts5_test_collist 116 140 fts5_test_tokenize 117 141 fts5_test_rowcount 118 142 fts5_test_all 119 143 120 144 fts5_test_queryphrase 121 145 fts5_test_phrasecount 122 146 } { ................................................................................ 174 198 # <phrase number> . <column number> . <token offset> 175 199 # 176 200 # Options: 177 201 # 178 202 # -near N (NEAR distance. Default 10) 179 203 # -col C (List of column indexes to match against) 180 204 # -pc VARNAME (variable in caller frame to use for phrase numbering) 205 +# -dict VARNAME (array in caller frame to use for synonyms) 181 206 # 182 207 proc nearset {aCol args} { 208 + 209 + # Process the command line options. 210 + # 183 211 set O(-near) 10 184 212 set O(-col) {} 185 213 set O(-pc) "" 214 + set O(-dict) "" 186 215 187 216 set nOpt [lsearch -exact $args --] 188 217 if {$nOpt<0} { error "no -- option" } 218 + 219 + # Set $lPhrase to be a list of phrases. $nPhrase its length. 220 + set lPhrase [lrange $args [expr $nOpt+1] end] 221 + set nPhrase [llength $lPhrase] 189 222 190 223 foreach {k v} [lrange $args 0 [expr $nOpt-1]] { 191 224 if {[info exists O($k)]==0} { error "unrecognized option $k" } 192 225 set O($k) $v 193 226 } 194 227 195 228 if {$O(-pc) == ""} { 196 229 set counter 0 197 230 } else { 198 231 upvar $O(-pc) counter 199 232 } 200 233 201 - # Set $phraselist to be a list of phrases. $nPhrase its length. 202 - set phraselist [lrange $args [expr $nOpt+1] end] 203 - set nPhrase [llength $phraselist] 234 + if {$O(-dict)!=""} { upvar $O(-dict) aDict } 204 235 205 236 for {set j 0} {$j < [llength $aCol]} {incr j} { 206 237 for {set i 0} {$i < $nPhrase} {incr i} { 207 238 set A($j,$i) [list] 208 239 } 209 240 } 210 241 211 - set iCol -1 212 - foreach col $aCol { 213 - incr iCol 242 + # Loop through each column of the current row. 243 + for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} { 244 + 245 + # If there is a column filter, test whether this column is excluded. If 246 + # so, skip to the next iteration of this loop. Otherwise, set zCol to the 247 + # column value and nToken to the number of tokens that comprise it. 214 248 if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue 215 - set nToken [llength $col] 249 + set zCol [lindex $aCol $iCol] 250 + set nToken [llength $zCol] 216 251 217 - set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] 218 - for { } {$iFL < $nToken} {incr iFL} { 219 - for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { 220 - set B($iPhrase) [list] 221 - } 252 + # Each iteration of the following loop searches a substring of the 253 + # column value for phrase matches. The last token of the substring 254 + # is token $iLast of the column value. The first token is: 255 + # 256 + # iFirst = ($iLast - $O(-near) - 1) 257 + # 258 + # where $sz is the length of the phrase being searched for. A phrase 259 + # counts as matching the substring if its first token lies on or before 260 + # $iLast and its last token on or after $iFirst. 261 + # 262 + # For example, if the query is "NEAR(a+b c, 2)" and the column value: 263 + # 264 + # "x x x x A B x x C x" 265 + # 0 1 2 3 4 5 6 7 8 9" 266 + # 267 + # when (iLast==8 && iFirst=5) the range will contain both phrases and 268 + # so both instances can be added to the output poslists. 269 + # 270 + set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)] 271 + for { } {$iLast < $nToken} {incr iLast} { 272 + 273 + catch { array unset B } 222 274 223 275 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { 224 - set p [lindex $phraselist $iPhrase] 276 + set p [lindex $lPhrase $iPhrase] 225 277 set nPm1 [expr {[llength $p] - 1}] 226 - set iFirst [expr $iFL - $O(-near) - [llength $p]] 278 + set iFirst [expr $iLast - $O(-near) - [llength $p]] 227 279 228 - for {set i $iFirst} {$i <= $iFL} {incr i} { 229 - if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i } 280 + for {set i $iFirst} {$i <= $iLast} {incr i} { 281 + set lCand [lrange $zCol $i [expr $i+$nPm1]] 282 + set bMatch 1 283 + foreach tok $p term $lCand { 284 + if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break } 285 + } 286 + if {$bMatch} { lappend B($iPhrase) $i } 230 287 } 231 - if {[llength $B($iPhrase)] == 0} break 288 + 289 + if {![info exists B($iPhrase)]} break 232 290 } 233 291 234 292 if {$iPhrase==$nPhrase} { 235 293 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { 236 294 set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)] 237 295 set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)] 238 296 } ................................................................................ 248 306 foreach a $A($iCol,$iPhrase) { 249 307 lappend res "$counter.$iCol.$a" 250 308 } 251 309 } 252 310 incr counter 253 311 } 254 312 255 - #puts $res 313 + #puts "$aCol -> $res" 256 314 sort_poslist $res 257 315 } 316 + 317 +proc nearset_match {aDictVar tok term} { 318 + if {[string match $tok $term]} { return 1 } 319 + 320 + upvar $aDictVar aDict 321 + if {[info exists aDict($tok)]} { 322 + foreach s $aDict($tok) { 323 + if {[string match $s $term]} { return 1 } 324 + } 325 + } 326 + return 0; 327 +} 258 328 259 329 #------------------------------------------------------------------------- 260 330 # Usage: 261 331 # 262 332 # sort_poslist LIST 263 333 # 264 334 # Sort a position list of the type returned by command [nearset] ................................................................................ 322 392 lappend ret $word $iOff [expr $iOff+$nToken] 323 393 incr iOff $nToken 324 394 incr iOff [gobble_whitespace text] 325 395 } 326 396 327 397 set ret 328 398 } 399 + 400 +#------------------------------------------------------------------------- 401 +# 402 +proc foreach_detail_mode {prefix script} { 403 + set saved $::testprefix 404 + foreach d [list full col none] { 405 + set s [string map [list %DETAIL% $d] $script] 406 + set ::detail $d 407 + set ::testprefix "$prefix-$d" 408 + reset_db 409 + uplevel $s 410 + unset ::detail 411 + } 412 + set ::testprefix $saved 413 +} 414 + 415 +proc detail_check {} { 416 + if {$::detail != "none" && $::detail!="full" && $::detail!="col"} { 417 + error "not in foreach_detail_mode {...} block" 418 + } 419 +} 420 +proc detail_is_none {} { detail_check ; expr {$::detail == "none"} } 421 +proc detail_is_col {} { detail_check ; expr {$::detail == "col" } } 422 +proc detail_is_full {} { detail_check ; expr {$::detail == "full"} } 423 + 424 + 425 +#------------------------------------------------------------------------- 426 +# Convert a poslist of the type returned by fts5_test_poslist() to a 427 +# collist as returned by fts5_test_collist(). 428 +# 429 +proc fts5_poslist2collist {poslist} { 430 + set res [list] 431 + foreach h $poslist { 432 + regexp {(.*)\.[1234567890]+} $h -> cand 433 + lappend res $cand 434 + } 435 + set res [lsort -command fts5_collist_elem_compare -unique $res] 436 + return $res 437 +} 438 + 439 +# Comparison function used by fts5_poslist2collist to sort collist entries. 440 +proc fts5_collist_elem_compare {a b} { 441 + foreach {a1 a2} [split $a .] {} 442 + foreach {b1 b2} [split $b .] {} 443 + 444 + if {$a1==$b1} { return [expr $a2 - $b2] } 445 + return [expr $a1 - $b1] 446 +} 447 + 448 + 449 +#-------------------------------------------------------------------------- 450 +# Construct and return a tcl list equivalent to that returned by the SQL 451 +# query executed against database handle [db]: 452 +# 453 +# SELECT 454 +# rowid, 455 +# fts5_test_poslist($tbl), 456 +# fts5_test_collist($tbl) 457 +# FROM $tbl('$expr') 458 +# ORDER BY rowid $order; 459 +# 460 +proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} { 461 + 462 + # Figure out the set of columns in the FTS5 table. This routine does 463 + # not handle tables with UNINDEXED columns, but if it did, it would 464 + # have to be here. 465 + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } 466 + 467 + set d "" 468 + if {$aDictVar != ""} { 469 + upvar $aDictVar aDict 470 + set d aDict 471 + } 472 + 473 + set cols "" 474 + foreach e $lCols { append cols ", '$e'" } 475 + set tclexpr [db one [subst -novar { 476 + SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] ) 477 + }]] 478 + 479 + set res [list] 480 + db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { 481 + set cols [list] 482 + foreach col $lCols { lappend cols $x($col) } 483 + 484 + set ::pc 0 485 + set rowdata [eval $tclexpr] 486 + if {$rowdata != ""} { 487 + lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata] 488 + } 489 + } 490 + 491 + set res 492 +} 493 + 494 +#------------------------------------------------------------------------- 495 +# Similar to [fts5_query_data], but omit the collist field. 496 +# 497 +proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} { 498 + set res [list] 499 + 500 + if {$aDictVar!=""} { 501 + upvar $aDictVar aDict 502 + set dict aDict 503 + } else { 504 + set dict "" 505 + } 506 + 507 + foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] { 508 + lappend res $rowid $poslist 509 + } 510 + set res 511 +} 512 + 513 +#------------------------------------------------------------------------- 514 +# 515 + 516 +# This command will only work inside a [foreach_detail_mode] block. It tests 517 +# whether or not expression $expr run on FTS5 table $tbl is supported by 518 +# the current mode. If so, 1 is returned. If not, 0. 519 +# 520 +# detail=full (all queries supported) 521 +# detail=col (all but phrase queries and NEAR queries) 522 +# detail=none (all but phrase queries, NEAR queries, and column filters) 523 +# 524 +proc fts5_expr_ok {expr tbl} { 525 + 526 + if {![detail_is_full]} { 527 + set nearset "nearset_rc" 528 + if {[detail_is_col]} { set nearset "nearset_rf" } 529 + 530 + set ::expr_not_ok 0 531 + db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) } 532 + 533 + set cols "" 534 + foreach e $lCols { append cols ", '$e'" } 535 + set ::pc 0 536 + set tclexpr [db one [subst -novar { 537 + SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] ) 538 + }]] 539 + eval $tclexpr 540 + if {$::expr_not_ok} { return 0 } 541 + } 542 + 543 + return 1 544 +} 545 + 546 +# Helper for [fts5_expr_ok] 547 +proc nearset_rf {aCol args} { 548 + set idx [lsearch -exact $args --] 549 + if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} { 550 + set ::expr_not_ok 1 551 + } 552 + list 553 +} 554 + 555 +# Helper for [fts5_expr_ok] 556 +proc nearset_rc {aCol args} { 557 + nearset_rf $aCol {*}$args 558 + if {[lsearch $args -col]>=0} { 559 + set ::expr_not_ok 1 560 + } 561 + list 562 +} 329 563
Changes to ext/fts5/test/fts5aa.test.
17 17 18 18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 19 ifcapable !fts5 { 20 20 finish_test 21 21 return 22 22 } 23 23 24 +foreach_detail_mode $::testprefix { 25 + 24 26 do_execsql_test 1.0 { 25 27 CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); 26 28 SELECT name, sql FROM sqlite_master; 27 29 } { 28 30 t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} 29 31 t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} 30 32 t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} ................................................................................ 37 39 DROP TABLE t1; 38 40 SELECT name, sql FROM sqlite_master; 39 41 } { 40 42 } 41 43 42 44 #------------------------------------------------------------------------- 43 45 # 44 -reset_db 46 + 45 47 do_execsql_test 2.0 { 46 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 48 + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); 47 49 } 48 50 do_execsql_test 2.1 { 49 51 INSERT INTO t1 VALUES('a b c', 'd e f'); 50 52 } 51 53 52 54 do_test 2.2 { 53 55 execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } ................................................................................ 61 63 SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC; 62 64 } {1} 63 65 } 64 66 65 67 do_execsql_test 2.4 { 66 68 INSERT INTO t1(t1) VALUES('integrity-check'); 67 69 } 70 + 68 71 69 72 #------------------------------------------------------------------------- 70 73 # 71 74 reset_db 72 75 do_execsql_test 3.0 { 73 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 76 + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); 74 77 } 75 78 foreach {i x y} { 76 79 1 {g f d b f} {h h e i a} 77 80 2 {f i g j e} {i j c f f} 78 81 3 {e e i f a} {e h f d f} 79 82 4 {h j f j i} {h a c f j} 80 83 5 {d b j c g} {f e i b e} ................................................................................ 89 92 if {[set_test_counter errors]} break 90 93 } 91 94 92 95 #------------------------------------------------------------------------- 93 96 # 94 97 reset_db 95 98 do_execsql_test 4.0 { 96 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 99 + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); 97 100 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 98 101 } 99 102 foreach {i x y} { 100 103 1 {g f d b f} {h h e i a} 101 104 2 {f i g j e} {i j c f f} 102 105 3 {e e i f a} {e h f d f} 103 106 4 {h j f j i} {h a c f j} ................................................................................ 113 116 if {[set_test_counter errors]} break 114 117 } 115 118 116 119 #------------------------------------------------------------------------- 117 120 # 118 121 reset_db 119 122 do_execsql_test 5.0 { 120 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 123 + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); 121 124 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 122 125 } 123 126 foreach {i x y} { 124 127 1 {dd abc abc abc abcde} {aaa dd ddd ddd aab} 125 128 2 {dd aab d aaa b} {abcde c aaa aaa aaa} 126 129 3 {abcde dd b b dd} {abc abc d abc ddddd} 127 130 4 {aaa abcde dddd dddd abcde} {abc b b abcde abc} ................................................................................ 137 140 if {[set_test_counter errors]} break 138 141 } 139 142 140 143 #------------------------------------------------------------------------- 141 144 # 142 145 reset_db 143 146 do_execsql_test 6.0 { 144 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 147 + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); 145 148 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 146 149 } 147 150 148 151 do_execsql_test 6.1 { 149 152 INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a'); 150 153 REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d'); 151 154 } ................................................................................ 272 275 } 273 276 274 277 275 278 #------------------------------------------------------------------------- 276 279 # 277 280 reset_db 278 281 do_execsql_test 10.0 { 279 - CREATE VIRTUAL TABLE t1 USING fts5(x,y); 282 + CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%); 280 283 } 281 284 set d10 { 282 285 1 {g f d b f} {h h e i a} 283 286 2 {f i g j e} {i j c f f} 284 287 3 {e e i f a} {e h f d f} 285 288 4 {h j f j i} {h a c f j} 286 289 5 {d b j c g} {f e i b e} ................................................................................ 305 308 306 309 do_execsql_test 10.4.1 { DELETE FROM t1 } 307 310 do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') } 308 311 309 312 #------------------------------------------------------------------------- 310 313 # 311 314 do_catchsql_test 11.1 { 312 - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank); 315 + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%); 313 316 } {1 {reserved fts5 column name: rank}} 314 317 do_catchsql_test 11.2 { 315 - CREATE VIRTUAL TABLE rank USING fts5(a, b, c); 318 + CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%); 316 319 } {1 {reserved fts5 table name: rank}} 317 320 do_catchsql_test 11.3 { 318 - CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid); 321 + CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%); 319 322 } {1 {reserved fts5 column name: rowid}} 320 323 321 324 #------------------------------------------------------------------------- 322 325 # 323 326 do_execsql_test 12.1 { 324 - CREATE VIRTUAL TABLE t2 USING fts5(x,y); 327 + CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%); 325 328 } {} 326 329 327 330 do_catchsql_test 12.2 { 328 331 SELECT t2 FROM t2 WHERE t2 MATCH '*stuff' 329 332 } {1 {unknown special query: stuff}} 330 333 331 334 do_test 12.3 { ................................................................................ 333 336 string is integer $res 334 337 } {1} 335 338 336 339 #------------------------------------------------------------------------- 337 340 # 338 341 reset_db 339 342 do_execsql_test 13.1 { 340 - CREATE VIRTUAL TABLE t1 USING fts5(x); 343 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 341 344 INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o'); 342 345 } {} 343 346 344 347 do_execsql_test 13.2 { 345 348 SELECT rowid FROM t1 WHERE t1 MATCH 'o'; 346 349 } {1 2} 347 350 ................................................................................ 357 360 SELECT rowid FROM t1 WHERE t1 MATCH '""'; 358 361 } {} 359 362 360 363 #------------------------------------------------------------------------- 361 364 # 362 365 reset_db 363 366 do_execsql_test 14.1 { 364 - CREATE VIRTUAL TABLE t1 USING fts5(x, y); 367 + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); 365 368 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 366 369 WITH d(x,y) AS ( 367 370 SELECT NULL, 'xyz xyz xyz xyz xyz xyz' 368 371 UNION ALL 369 372 SELECT NULL, 'xyz xyz xyz xyz xyz xyz' FROM d 370 373 ) 371 374 INSERT INTO t1 SELECT * FROM d LIMIT 200; 372 375 } 376 + 377 +do_execsql_test 15.x { 378 + INSERT INTO t1(t1) VALUES('integrity-check'); 379 +} 373 380 374 381 do_test 14.2 { 375 382 set nRow 0 376 383 db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } { 377 384 db eval { 378 385 BEGIN; 379 386 CREATE TABLE t2(a, b); ................................................................................ 413 420 # 414 421 do_execsql_test 16.1 { 415 422 CREATE VIRTUAL TABLE n1 USING fts5(a); 416 423 INSERT INTO n1 VALUES('a b c d'); 417 424 } 418 425 419 426 proc funk {} { 427 + db eval { UPDATE n1_config SET v=50 WHERE k='version' } 420 428 set fd [db incrblob main n1_data block 10] 421 429 fconfigure $fd -encoding binary -translation binary 422 430 puts -nonewline $fd "\x44\x45" 423 431 close $fd 424 - db eval { UPDATE n1_config SET v=50 WHERE k='version' } 425 432 } 426 433 db func funk funk 427 434 428 435 do_catchsql_test 16.2 { 429 436 SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d' 430 437 } {1 {SQL logic error or missing database}} 431 438 432 439 #------------------------------------------------------------------------- 433 440 # 434 441 reset_db 435 442 do_execsql_test 17.1 { 436 - CREATE VIRTUAL TABLE b2 USING fts5(x); 443 + CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%); 437 444 INSERT INTO b2 VALUES('a'); 438 445 INSERT INTO b2 VALUES('b'); 439 446 INSERT INTO b2 VALUES('c'); 440 447 } 441 448 442 449 do_test 17.2 { 443 450 set res [list] 444 451 db eval { SELECT * FROM b2 ORDER BY rowid ASC } { 445 452 lappend res [execsql { SELECT * FROM b2 ORDER BY rowid ASC }] 446 453 } 447 454 set res 448 455 } {{a b c} {a b c} {a b c}} 449 456 450 -reset_db 451 -do_execsql_test 18.1 { 452 - CREATE VIRTUAL TABLE c2 USING fts5(x, y); 453 - INSERT INTO c2 VALUES('x x x', 'x x x'); 454 - SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; 455 -} {1} 457 +if {[string match n* %DETAIL%]==0} { 458 + reset_db 459 + do_execsql_test 17.3 { 460 + CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%); 461 + INSERT INTO c2 VALUES('x x x', 'x x x'); 462 + SELECT rowid FROM c2 WHERE c2 MATCH 'y:x'; 463 + } {1} 464 +} 456 465 457 466 #------------------------------------------------------------------------- 458 467 # 459 468 reset_db 460 469 do_execsql_test 17.1 { 461 - CREATE VIRTUAL TABLE uio USING fts5(ttt); 470 + CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%); 462 471 INSERT INTO uio VALUES(NULL); 463 472 INSERT INTO uio SELECT NULL FROM uio; 464 473 INSERT INTO uio SELECT NULL FROM uio; 465 474 INSERT INTO uio SELECT NULL FROM uio; 466 475 INSERT INTO uio SELECT NULL FROM uio; 467 476 INSERT INTO uio SELECT NULL FROM uio; 468 477 INSERT INTO uio SELECT NULL FROM uio; ................................................................................ 501 510 do_execsql_test 17.9 { 502 511 SELECT min(rowid), max(rowid), count(*) FROM uio WHERE rowid < 10; 503 512 } {-9223372036854775808 9 10} 504 513 505 514 #-------------------------------------------------------------------- 506 515 # 507 516 do_execsql_test 18.1 { 508 - CREATE VIRTUAL TABLE t1 USING fts5(a, b); 509 - CREATE VIRTUAL TABLE t2 USING fts5(c, d); 517 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 518 + CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%); 510 519 INSERT INTO t1 VALUES('abc*', NULL); 511 520 INSERT INTO t2 VALUES(1, 'abcdefg'); 512 521 } 513 522 do_execsql_test 18.2 { 514 523 SELECT t1.rowid, t2.rowid FROM t1, t2 WHERE t2 MATCH t1.a AND t1.rowid = t2.c 515 524 } {1 1} 516 525 do_execsql_test 18.3 { ................................................................................ 518 527 } {1 1} 519 528 520 529 #-------------------------------------------------------------------- 521 530 # fts5 table in the temp schema. 522 531 # 523 532 reset_db 524 533 do_execsql_test 19.0 { 525 - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); 534 + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); 526 535 INSERT INTO t1 VALUES('x y z'); 527 536 INSERT INTO t1 VALUES('w x 1'); 528 537 SELECT rowid FROM t1 WHERE t1 MATCH 'x'; 529 538 } {1 2} 530 539 531 540 #-------------------------------------------------------------------- 532 541 # Test that 6 and 7 byte varints can be read. 533 542 # 534 543 reset_db 535 544 do_execsql_test 20.0 { 536 - CREATE VIRTUAL TABLE temp.tmp USING fts5(x); 545 + CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%); 537 546 } 538 547 set ::ids [list \ 539 548 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43] 540 549 ] 541 550 do_test 20.1 { 542 551 foreach id $::ids { 543 552 execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') } 544 553 } 545 554 execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' } 546 555 } $::ids 547 556 557 +} 548 558 549 559 550 560 finish_test 551 561 552 562
Changes to ext/fts5/test/fts5ab.test.
18 18 19 19 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 20 20 ifcapable !fts5 { 21 21 finish_test 22 22 return 23 23 } 24 24 25 +foreach_detail_mode $testprefix { 26 + 25 27 do_execsql_test 1.0 { 26 - CREATE VIRTUAL TABLE t1 USING fts5(a, b); 28 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 27 29 INSERT INTO t1 VALUES('hello', 'world'); 28 30 INSERT INTO t1 VALUES('one two', 'three four'); 29 31 INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five'); 30 32 } 31 33 32 34 do_execsql_test 1.1 { 33 35 SELECT * FROM t1 ORDER BY rowid DESC; ................................................................................ 53 55 SELECT * FROM t1 WHERE rowid=1.99; 54 56 } {} 55 57 56 58 #------------------------------------------------------------------------- 57 59 58 60 reset_db 59 61 do_execsql_test 2.1 { 60 - CREATE VIRTUAL TABLE t1 USING fts5(x); 62 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 61 63 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 62 64 INSERT INTO t1 VALUES('one'); 63 65 INSERT INTO t1 VALUES('two'); 64 66 INSERT INTO t1 VALUES('three'); 65 67 } 66 68 67 69 do_catchsql_test 2.2 { ................................................................................ 155 157 } 156 158 157 159 #------------------------------------------------------------------------- 158 160 # Documents with more than 2M tokens. 159 161 # 160 162 161 163 do_execsql_test 4.0 { 162 - CREATE VIRTUAL TABLE s1 USING fts5(x); 164 + CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%); 163 165 } 164 166 foreach {tn doc} [list \ 165 167 1 [string repeat {a x } 1500000] \ 166 168 2 "[string repeat {a a } 1500000] x" \ 167 169 ] { 168 170 do_execsql_test 4.$tn { INSERT INTO s1 VALUES($doc) } 169 171 } 170 172 171 173 do_execsql_test 4.3 { 172 174 SELECT rowid FROM s1 WHERE s1 MATCH 'x' 173 175 } {1 2} 174 176 175 -do_execsql_test 4.4 { 176 - SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' 177 +if {[detail_is_full]} { 178 + do_execsql_test 4.4 { 179 + SELECT rowid FROM s1 WHERE s1 MATCH '"a x"' 180 + } {1 2} 181 +} 182 + 183 +do_execsql_test 4.5 { 184 + SELECT rowid FROM s1 WHERE s1 MATCH 'a x' 177 185 } {1 2} 178 186 179 187 #------------------------------------------------------------------------- 180 188 # Check that a special case of segment promotion works. The case is where 181 189 # a new segment is written to level L, but the oldest segment within level 182 190 # (L-2) is larger than it. 183 191 # 184 192 do_execsql_test 5.0 { 185 - CREATE VIRTUAL TABLE s2 USING fts5(x); 193 + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); 186 194 INSERT INTO s2(s2, rank) VALUES('pgsz', 32); 187 195 INSERT INTO s2(s2, rank) VALUES('automerge', 0); 188 196 } 189 197 190 198 proc rnddoc {n} { 191 199 set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] 192 200 set doc [list] ................................................................................ 218 226 } {8 0 0} 219 227 220 228 # Test also the other type of segment promotion - when a new segment is written 221 229 # that is larger than segments immediately following it. 222 230 do_test 5.3 { 223 231 execsql { 224 232 DROP TABLE s2; 225 - CREATE VIRTUAL TABLE s2 USING fts5(x); 233 + CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%); 226 234 INSERT INTO s2(s2, rank) VALUES('pgsz', 32); 227 235 INSERT INTO s2(s2, rank) VALUES('automerge', 0); 228 236 } 229 237 230 238 for {set i 1} {$i <= 16} {incr i} { 231 239 execsql { INSERT INTO s2 VALUES(rnddoc(5)) } 232 240 } ................................................................................ 237 245 execsql { INSERT INTO s2 VALUES(rnddoc(160)) } 238 246 fts5_level_segs s2 239 247 } {2 0} 240 248 241 249 #------------------------------------------------------------------------- 242 250 # 243 251 do_execsql_test 6.0 { 244 - CREATE VIRTUAL TABLE s3 USING fts5(x); 252 + CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%); 245 253 BEGIN; 246 254 INSERT INTO s3 VALUES('a b c'); 247 255 INSERT INTO s3 VALUES('A B C'); 248 256 } 249 257 250 258 do_execsql_test 6.1.1 { 251 259 SELECT rowid FROM s3 WHERE s3 MATCH 'a' ................................................................................ 272 280 ROLLBACK; 273 281 } 274 282 } {} 275 283 276 284 #------------------------------------------------------------------------- 277 285 # 278 286 set doc [string repeat "a b c " 500] 279 -breakpoint 280 287 do_execsql_test 7.0 { 281 - CREATE VIRTUAL TABLE x1 USING fts5(x); 288 + CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%); 282 289 INSERT INTO x1(x1, rank) VALUES('pgsz', 32); 283 290 INSERT INTO x1 VALUES($doc); 284 291 } 285 292 293 +} ;# foreach_detail_mode... 286 294 287 295 288 296 finish_test 289 297
Changes to ext/fts5/test/fts5ac.test.
18 18 19 19 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 20 20 ifcapable !fts5 { 21 21 finish_test 22 22 return 23 23 } 24 24 25 +foreach_detail_mode $testprefix { 26 + 25 27 set data { 26 28 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 27 29 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} 28 30 2 {l s z j k i m p s} {l w e j t j e e i t w r o p o} 29 31 3 {x g y m y m h p} {k j j b r e y y a k y} 30 32 4 {q m a i y i z} {o w a g k x g j m w e u k} 31 33 5 {k o a w y b s z} {s g l m m l m g p} ................................................................................ 120 122 94 {s f e a e t i h h d q p z t q} {b k m k w h c} 121 123 95 {h b n j t k i h o q u} {w n g i t o k c a m y p f l x c p} 122 124 96 {f c x p y r b m o l m o a} {p c a q s u n n x d c f a o} 123 125 97 {u h h k m n k} {u b v n u a o c} 124 126 98 {s p e t c z d f n w f} {l s f j b l c e s h} 125 127 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} 126 128 } 127 - 128 -# Argument $expr is an FTS5 match expression designed to be executed against 129 -# an FTS5 table with the following schema: 130 -# 131 -# CREATE VIRTUAL TABLE xy USING fts5(x, y); 132 -# 133 -# Assuming the table contains the same records as stored int the global 134 -# $::data array (see above), this function returns a list containing one 135 -# element for each match in the dataset. The elements are themselves lists 136 -# formatted as follows: 137 -# 138 -# <rowid> {<phrase 0 matches> <phrase 1 matches>...} 139 -# 140 -# where each <phrase X matches> element is a list of phrase matches in the 141 -# same form as returned by auxiliary scalar function fts5_test(). 142 -# 143 -proc matchdata {bPos expr {bAsc 1}} { 144 - 145 - set tclexpr [db one { 146 - SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y') 147 - }] 148 - set res [list] 149 - 150 - #puts $tclexpr 151 - foreach {id x y} $::data { 152 - set cols [list $x $y] 153 - set ::pc 0 154 - #set hits [lsort -command instcompare [eval $tclexpr]] 155 - set hits [eval $tclexpr] 156 - if {[llength $hits]>0} { 157 - if {$bPos} { 158 - lappend res [list $id $hits] 159 - } else { 160 - lappend res $id 161 - } 162 - } 163 - } 164 - 165 - if {$bAsc} { 166 - set res [lsort -integer -increasing -index 0 $res] 167 - } else { 168 - set res [lsort -integer -decreasing -index 0 $res] 169 - } 170 - 171 - return [concat {*}$res] 172 -} 173 - 174 -# 175 -# End of test code 176 -#------------------------------------------------------------------------- 177 - 178 -proc fts5_test_poslist {cmd} { 179 - set res [list] 180 - for {set i 0} {$i < [$cmd xInstCount]} {incr i} { 181 - lappend res [string map {{ } .} [$cmd xInst $i]] 182 - } 183 - set res 184 -} 185 - 186 129 187 130 foreach {tn2 sql} { 188 131 1 {} 189 132 2 {BEGIN} 190 133 } { 191 134 reset_db 192 - sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist 135 + fts5_aux_test_functions db 193 136 194 - do_execsql_test 1.0 { 195 - CREATE VIRTUAL TABLE xx USING fts5(x,y); 137 + do_execsql_test 1.$tn2.0 { 138 + CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%); 196 139 INSERT INTO xx(xx, rank) VALUES('pgsz', 32); 197 140 } 198 141 199 142 execsql $sql 200 143 201 - do_test $tn2.1.1 { 144 + do_test 1.$tn2.1.1 { 202 145 foreach {id x y} $data { 203 146 execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } 204 147 } 205 148 execsql { INSERT INTO xx(xx) VALUES('integrity-check') } 206 149 } {} 207 150 208 151 209 152 #------------------------------------------------------------------------- 210 - # Test phrase queries. 211 153 # 212 - foreach {tn phrase} { 213 - 1 "o" 214 - 2 "b q" 215 - 3 "e a e" 216 - 4 "m d g q q b k b w f q q p p" 217 - 5 "l o o l v v k" 218 - 6 "a" 219 - 7 "b" 220 - 8 "c" 221 - 9 "no" 222 - 10 "L O O L V V K" 223 - } { 224 - set expr "\"$phrase\"" 225 - set res [matchdata 1 $expr] 226 - 227 - do_execsql_test $tn2.1.2.$tn.[llength $res] { 228 - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 229 - } $res 230 - } 231 - 232 - #------------------------------------------------------------------------- 233 - # Test some AND and OR queries. 234 - # 235 - foreach {tn expr} { 236 - 1.1 "a AND b" 237 - 1.2 "a+b AND c" 238 - 1.3 "d+c AND u" 239 - 1.4 "d+c AND u+d" 240 - 241 - 2.1 "a OR b" 242 - 2.2 "a+b OR c" 243 - 2.3 "d+c OR u" 244 - 2.4 "d+c OR u+d" 245 - 246 - 3.1 { a AND b AND c } 247 - } { 248 - set res [matchdata 1 $expr] 249 - do_execsql_test $tn2.2.$tn.[llength $res] { 250 - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 251 - } $res 252 - } 253 - 254 - #------------------------------------------------------------------------- 255 - # Queries on a specific column. 256 - # 257 - foreach {tn expr} { 258 - 1.1 "x:a" 259 - 1.2 "y:a" 260 - 1.3 "x:b" 261 - 1.4 "y:b" 262 - 2.1 "{x}:a" 263 - 2.2 "{y}:a" 264 - 2.3 "{x}:b" 265 - 2.4 "{y}:b" 266 - 267 - 3.1 "{x y}:a" 268 - 3.2 "{y x}:a" 269 - 3.3 "{x x}:b" 270 - 3.4 "{y y}:b" 271 - 272 - 4.1 {{"x" "y"}:a} 273 - 4.2 {{"y" x}:a} 274 - 4.3 {{x "x"}:b} 275 - 4.4 {{"y" y}:b} 276 - } { 277 - set res [matchdata 1 $expr] 278 - do_execsql_test $tn2.3.$tn.[llength $res] { 279 - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 280 - } $res 281 - } 282 - 283 - #------------------------------------------------------------------------- 284 - # Some NEAR queries. 285 - # 286 - foreach {tn expr} { 287 - 1 "NEAR(a b)" 288 - 2 "NEAR(r c)" 289 - 2 { NEAR(r c, 5) } 290 - 3 { NEAR(r c, 3) } 291 - 4 { NEAR(r c, 2) } 292 - 5 { NEAR(r c, 0) } 293 - 6 { NEAR(a b c) } 294 - 7 { NEAR(a b c, 8) } 295 - 8 { x : NEAR(r c) } 296 - 9 { y : NEAR(r c) } 297 - } { 298 - set res [matchdata 1 $expr] 299 - do_execsql_test $tn2.4.1.$tn.[llength $res] { 300 - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 301 - } $res 302 - } 303 - 304 - do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0} 305 - do_test $tn2.4.2 { nearset {{a b c}} -- c } {0.0.2} 306 - 307 - foreach {tn expr tclexpr} { 308 - 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} 309 - } { 310 - do_execsql_test $tn2.5.$tn { 311 - SELECT fts5_expr_tcl($expr, 'N $x') 312 - } [list $tclexpr] 154 + do_execsql_test 1.$tn2.integrity { 155 + INSERT INTO xx(xx) VALUES('integrity-check'); 313 156 } 314 157 315 158 #------------------------------------------------------------------------- 316 159 # 317 - do_execsql_test $tn2.6.integrity { 318 - INSERT INTO xx(xx) VALUES('integrity-check'); 319 - } 320 - #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} 321 - foreach {bAsc sql} { 322 - 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} 323 - 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} 160 + foreach {tn expr} { 161 + 1.2 "a OR b" 162 + 1.1 "a AND b" 163 + 1.3 "o" 164 + 1.4 "b q" 165 + 1.5 "e a e" 166 + 1.6 "m d g q q b k b w f q q p p" 167 + 1.7 "l o o l v v k" 168 + 1.8 "a" 169 + 1.9 "b" 170 + 1.10 "c" 171 + 1.11 "no" 172 + 1.12 "L O O L V V K" 173 + 1.13 "a AND b AND c" 174 + 1.14 "x:a" 175 + 176 + 2.1 "x:a" 177 + 2.2 "y:a" 178 + 2.3 "x:b" 179 + 2.4 "y:b" 180 + 181 + 3.1 "{x}:a" 182 + 3.2 "{y}:a" 183 + 3.3 "{x}:b" 184 + 3.4 "{y}:b" 185 + 186 + 4.1 "{x y}:a" 187 + 4.2 "{y x}:a" 188 + 4.3 "{x x}:b" 189 + 4.4 "{y y}:b" 190 + 191 + 5.1 {{"x" "y"}:a} 192 + 5.2 {{"y" x}:a} 193 + 5.3 {{x "x"}:b} 194 + 5.4 {{"y" y}:b} 195 + 196 + 6.1 "b + q" 197 + 6.2 "e + a + e" 198 + 6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p" 199 + 6.4 "l + o + o + l + v + v + k" 200 + 6.5 "L + O + O + L + V + V + K" 201 + 202 + 7.1 "a+b AND c" 203 + 7.2 "d+c AND u" 204 + 7.3 "d+c AND u+d" 205 + 7.4 "a+b OR c" 206 + 7.5 "d+c OR u" 207 + 7.6 "d+c OR u+d" 208 + 209 + 8.1 "NEAR(a b)" 210 + 8.2 "NEAR(r c)" 211 + 8.2 { NEAR(r c, 5) } 212 + 8.3 { NEAR(r c, 3) } 213 + 8.4 { NEAR(r c, 2) } 214 + 8.5 { NEAR(r c, 0) } 215 + 8.6 { NEAR(a b c) } 216 + 8.7 { NEAR(a b c, 8) } 217 + 8.8 { x : NEAR(r c) } 218 + 8.9 { y : NEAR(r c) } 219 + 220 + 9.1 { NEAR(r c) } 221 + 9.2 { NEAR(r c, 5) } 222 + 9.3 { NEAR(r c, 3) } 223 + 9.4 { NEAR(r c, 2) } 224 + 9.5 { NEAR(r c, 0) } 225 + 9.6 { NEAR(a b c) } 226 + 9.7 { NEAR(a b c, 8) } 227 + 9.8 { x : NEAR(r c) } 228 + 9.9 { y : NEAR(r c) } 229 + 9.10 { x : "r c" } 230 + 9.11 { y : "r c" } 231 + 9.12 { a AND b } 232 + 9.13 { a AND b AND c } 233 + 9.14a { a } 234 + 9.14b { a OR b } 235 + 9.15 { a OR b AND c } 236 + 9.16 { c AND b OR a } 237 + 9.17 { c AND (b OR a) } 238 + 9.18 { c NOT (b OR a) } 239 + 9.19 { (c NOT b) OR (a AND d) } 324 240 } { 325 - foreach {tn expr} { 326 - 0.1 x 327 - 1 { NEAR(r c) } 328 - 2 { NEAR(r c, 5) } 329 - 3 { NEAR(r c, 3) } 330 - 4 { NEAR(r c, 2) } 331 - 5 { NEAR(r c, 0) } 332 - 6 { NEAR(a b c) } 333 - 7 { NEAR(a b c, 8) } 334 - 8 { x : NEAR(r c) } 335 - 9 { y : NEAR(r c) } 336 - 10 { x : "r c" } 337 - 11 { y : "r c" } 338 - 12 { a AND b } 339 - 13 { a AND b AND c } 340 - 14a { a } 341 - 14b { a OR b } 342 - 15 { a OR b AND c } 343 - 16 { c AND b OR a } 344 - 17 { c AND (b OR a) } 345 - 18 { c NOT (b OR a) } 346 - 19 { c NOT b OR a AND d } 347 - } { 348 - set res [matchdata 0 $expr $bAsc] 349 - do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res 241 + 242 + if {[fts5_expr_ok $expr xx]==0} { 243 + do_test 1.$tn2.$tn.OMITTED { list } [list] 244 + continue 350 245 } 246 + 247 + set res [fts5_query_data $expr xx] 248 + do_execsql_test 1.$tn2.$tn.[llength $res].asc { 249 + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 250 + FROM xx WHERE xx match $expr 251 + } $res 252 + 253 + 254 + set res [fts5_query_data $expr xx DESC] 255 + do_execsql_test 1.$tn2.$tn.[llength $res].desc { 256 + SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 257 + FROM xx WHERE xx match $expr ORDER BY 1 DESC 258 + } $res 351 259 } 352 260 } 353 261 354 -do_execsql_test 3.1 { 262 +} 263 + 264 +do_execsql_test 2.1 { 355 265 SELECT fts5_expr_tcl('a AND b'); 356 266 } {{AND [nearset -- {a}] [nearset -- {b}]}} 267 + 268 +do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0} 269 +do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2} 270 + 271 +foreach {tn expr tclexpr} { 272 + 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} 273 +} { 274 + do_execsql_test 2.3.$tn { 275 + SELECT fts5_expr_tcl($expr, 'N $x') 276 + } [list $tclexpr] 277 +} 357 278 358 279 finish_test 359 280
Changes to ext/fts5/test/fts5ad.test.
18 18 19 19 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 20 20 ifcapable !fts5 { 21 21 finish_test 22 22 return 23 23 } 24 24 25 +foreach_detail_mode $testprefix { 26 + 25 27 do_execsql_test 1.0 { 26 - CREATE VIRTUAL TABLE yy USING fts5(x, y); 28 + CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%); 27 29 INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching'); 28 30 INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is'); 29 31 INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the'); 30 32 } {} 31 33 32 34 foreach {tn match res} { 33 35 1 {c*} {1} ................................................................................ 49 51 do_execsql_test 1.$tn { 50 52 SELECT rowid FROM yy WHERE yy MATCH $match 51 53 } $res 52 54 } 53 55 54 56 foreach {T create} { 55 57 2 { 56 - CREATE VIRTUAL TABLE t1 USING fts5(a, b); 58 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 57 59 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 58 60 } 59 61 60 62 3 { 61 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); 63 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); 62 64 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 63 65 } 64 66 65 67 4 { 66 - CREATE VIRTUAL TABLE t1 USING fts5(a, b); 68 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 67 69 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 68 70 BEGIN; 69 71 } 70 72 71 73 5 { 72 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); 74 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%); 73 75 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 74 76 BEGIN; 75 77 } 76 78 77 79 } { 78 80 79 81 do_test $T.1 { ................................................................................ 229 231 set n [llength $res] 230 232 if {$T==5} breakpoint 231 233 do_execsql_test $T.$bAsc.$tn.$n $sql $res 232 234 } 233 235 } 234 236 235 237 catchsql COMMIT 238 +} 239 + 236 240 } 237 241 238 242 finish_test 239 243
Changes to ext/fts5/test/fts5ae.test.
18 18 19 19 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 20 20 ifcapable !fts5 { 21 21 finish_test 22 22 return 23 23 } 24 24 25 +foreach_detail_mode $testprefix { 26 + 25 27 do_execsql_test 1.0 { 26 - CREATE VIRTUAL TABLE t1 USING fts5(a, b); 28 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 27 29 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 28 30 } 29 31 30 32 do_execsql_test 1.1 { 31 33 INSERT INTO t1 VALUES('hello', 'world'); 32 34 SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; 33 35 } {1} ................................................................................ 51 53 } {1 2 4} 52 54 53 55 fts5_aux_test_functions db 54 56 55 57 #------------------------------------------------------------------------- 56 58 # 57 59 do_execsql_test 2.0 { 58 - CREATE VIRTUAL TABLE t2 USING fts5(x, y); 60 + CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%); 59 61 INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e'); 60 62 INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p'); 61 63 } 62 64 63 65 do_execsql_test 2.1 { 64 66 SELECT rowid, fts5_test_poslist(t2) FROM t2 65 67 WHERE t2 MATCH 'm' ORDER BY rowid; ................................................................................ 72 74 SELECT rowid, fts5_test_poslist(t2) FROM t2 73 75 WHERE t2 MATCH 'u OR q' ORDER BY rowid; 74 76 } { 75 77 1 {0.0.0} 76 78 2 {1.0.2 1.0.10} 77 79 } 78 80 79 -do_execsql_test 2.3 { 80 - SELECT rowid, fts5_test_poslist(t2) FROM t2 81 - WHERE t2 MATCH 'y:o' ORDER BY rowid; 82 -} { 83 - 1 {0.1.3 0.1.7} 81 +if {[detail_is_full]} { 82 + do_execsql_test 2.3 { 83 + SELECT rowid, fts5_test_poslist(t2) FROM t2 84 + WHERE t2 MATCH 'y:o' ORDER BY rowid; 85 + } { 86 + 1 {0.1.3 0.1.7} 87 + } 84 88 } 85 89 86 90 #------------------------------------------------------------------------- 87 91 # 88 92 do_execsql_test 3.0 { 89 - CREATE VIRTUAL TABLE t3 USING fts5(x, y); 93 + CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%); 90 94 INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t'); 91 95 INSERT INTO t3 VALUES( 'r c', ''); 92 96 } 93 97 94 -do_execsql_test 3.1 { 95 - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; 96 -} { 97 - 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} 98 -} 98 +if {[detail_is_full]} { 99 + do_execsql_test 3.1 { 100 + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)'; 101 + } { 102 + 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15} 103 + } 99 104 100 -do_execsql_test 3.2 { 101 - SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; 102 -} { 103 - 2 {0.0.0 1.0.1} 105 + do_execsql_test 3.2 { 106 + SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)'; 107 + } { 108 + 2 {0.0.0 1.0.1} 109 + } 104 110 } 105 111 106 112 do_execsql_test 3.3 { 107 113 INSERT INTO t3 108 114 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); 109 115 SELECT rowid, fts5_test_poslist(t3) 110 116 FROM t3 WHERE t3 MATCH 'a OR b AND c'; ................................................................................ 112 118 1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2} 113 119 3 0.0.5 114 120 } 115 121 116 122 #------------------------------------------------------------------------- 117 123 # 118 124 do_execsql_test 4.0 { 119 - CREATE VIRTUAL TABLE t4 USING fts5(x, y); 125 + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); 120 126 INSERT INTO t4 121 127 VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o'); 122 128 } 123 129 124 130 do_execsql_test 4.1 { 125 131 SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c'; 126 132 } { ................................................................................ 130 136 #------------------------------------------------------------------------- 131 137 # Test that the xColumnSize() and xColumnAvgsize() APIs work. 132 138 # 133 139 reset_db 134 140 fts5_aux_test_functions db 135 141 136 142 do_execsql_test 5.1 { 137 - CREATE VIRTUAL TABLE t5 USING fts5(x, y); 143 + CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%); 138 144 INSERT INTO t5 VALUES('a b c d', 'e f g h i j'); 139 145 INSERT INTO t5 VALUES('', 'a'); 140 146 INSERT INTO t5 VALUES('a', ''); 141 147 } 142 148 do_execsql_test 5.2 { 143 149 SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a' 144 150 ORDER BY rowid DESC; ................................................................................ 178 184 179 185 #------------------------------------------------------------------------- 180 186 # Test the xTokenize() API 181 187 # 182 188 reset_db 183 189 fts5_aux_test_functions db 184 190 do_execsql_test 6.1 { 185 - CREATE VIRTUAL TABLE t6 USING fts5(x, y); 191 + CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%); 186 192 INSERT INTO t6 VALUES('There are more', 'things in heaven and earth'); 187 193 INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.'); 188 194 } 189 195 190 196 do_execsql_test 6.2 { 191 197 SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*' 192 198 } { ................................................................................ 196 202 197 203 #------------------------------------------------------------------------- 198 204 # Test the xQueryPhrase() API 199 205 # 200 206 reset_db 201 207 fts5_aux_test_functions db 202 208 do_execsql_test 7.1 { 203 - CREATE VIRTUAL TABLE t7 USING fts5(x, y); 209 + CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%); 204 210 } 205 211 do_test 7.2 { 206 212 foreach {x y} { 207 213 {q i b w s a a e l o} {i b z a l f p t e u} 208 214 {b a z t a l o x d i} {b p a d b f h d w y} 209 215 {z m h n p p u i e g} {v h d v b x j j c z} 210 216 {a g i m v a u c b i} {p k s o t l r t b m} ................................................................................ 236 242 # SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a'; 237 243 #} {5 5 5 5} 238 244 # 239 245 240 246 #------------------------------------------------------------------------- 241 247 # 242 248 do_test 8.1 { 243 - execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) } 249 + execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) } 244 250 foreach {rowid x y} { 245 251 0 {A o} {o o o C o o o o o o o o} 246 252 1 {o o B} {o o o C C o o o o o o o} 247 253 2 {A o o} {o o o o D D o o o o o o} 248 254 3 {o B} {o o o o o D o o o o o o} 249 255 4 {E o G} {H o o o o o o o o o o o} 250 256 5 {F o G} {I o J o o o o o o o o o} ................................................................................ 294 300 2 {a OR b} 2 295 301 3 {a OR b OR c} 3 296 302 4 {NEAR(a b)} 2 297 303 } { 298 304 do_execsql_test 9.2.$tn { 299 305 SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1 300 306 } $cnt 307 +} 308 + 301 309 } 302 310 303 311 finish_test 304 312
Changes to ext/fts5/test/fts5af.test.
20 20 21 21 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 22 22 ifcapable !fts5 { 23 23 finish_test 24 24 return 25 25 } 26 26 27 +foreach_detail_mode $testprefix { 27 28 28 29 do_execsql_test 1.0 { 29 - CREATE VIRTUAL TABLE t1 USING fts5(x, y); 30 + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%); 30 31 } 31 32 32 33 proc do_snippet_test {tn doc match res} { 33 34 34 35 uplevel #0 [list set v1 $doc] 35 36 uplevel #0 [list set v2 $match] 36 37 ................................................................................ 107 108 7.4 {o o o X o o X o o} {...o [X] o o [X] o o} 108 109 7.5 {o o o o X o o X o} {...o o [X] o o [X] o} 109 110 7.6 {o o o o o X o o X} {...o o o [X] o o [X]} 110 111 } { 111 112 do_snippet_test 1.$tn $doc X $res 112 113 } 113 114 114 -foreach {tn doc res} { 115 - 1.1 {X Y o o o o o} {[X Y] o o o o o} 116 - 1.2 {o X Y o o o o} {o [X Y] o o o o} 117 - 1.3 {o o X Y o o o} {o o [X Y] o o o} 118 - 1.4 {o o o X Y o o} {o o o [X Y] o o} 119 - 1.5 {o o o o X Y o} {o o o o [X Y] o} 120 - 1.6 {o o o o o X Y} {o o o o o [X Y]} 115 +if {[detail_is_full]} { 116 + foreach {tn doc res} { 117 + 1.1 {X Y o o o o o} {[X Y] o o o o o} 118 + 1.2 {o X Y o o o o} {o [X Y] o o o o} 119 + 1.3 {o o X Y o o o} {o o [X Y] o o o} 120 + 1.4 {o o o X Y o o} {o o o [X Y] o o} 121 + 1.5 {o o o o X Y o} {o o o o [X Y] o} 122 + 1.6 {o o o o o X Y} {o o o o o [X Y]} 121 123 122 - 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 123 - 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 124 - 2.3 {o o X Y o o o o} {o o [X Y] o o o...} 125 - 2.4 {o o o X Y o o o} {...o o [X Y] o o o} 126 - 2.5 {o o o o X Y o o} {...o o o [X Y] o o} 127 - 2.6 {o o o o o X Y o} {...o o o o [X Y] o} 128 - 2.7 {o o o o o o X Y} {...o o o o o [X Y]} 124 + 2.1 {X Y o o o o o o} {[X Y] o o o o o...} 125 + 2.2 {o X Y o o o o o} {o [X Y] o o o o...} 126 + 2.3 {o o X Y o o o o} {o o [X Y] o o o...} 127 + 2.4 {o o o X Y o o o} {...o o [X Y] o o o} 128 + 2.5 {o o o o X Y o o} {...o o o [X Y] o o} 129 + 2.6 {o o o o o X Y o} {...o o o o [X Y] o} 130 + 2.7 {o o o o o o X Y} {...o o o o o [X Y]} 129 131 130 - 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 131 - 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 132 - 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} 133 - 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 134 - 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 135 - 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 136 - 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 137 - 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} 138 - 139 -} { 140 - do_snippet_test 2.$tn $doc "X + Y" $res 132 + 3.1 {X Y o o o o o o o} {[X Y] o o o o o...} 133 + 3.2 {o X Y o o o o o o} {o [X Y] o o o o...} 134 + 3.3 {o o X Y o o o o o} {o o [X Y] o o o...} 135 + 3.4 {o o o X Y o o o o} {...o o [X Y] o o o...} 136 + 3.5 {o o o o X Y o o o} {...o o [X Y] o o o} 137 + 3.6 {o o o o o X Y o o} {...o o o [X Y] o o} 138 + 3.7 {o o o o o o X Y o} {...o o o o [X Y] o} 139 + 3.8 {o o o o o o o X Y} {...o o o o o [X Y]} 140 + } { 141 + do_snippet_test 2.$tn $doc "X + Y" $res 142 + } 141 143 } 144 + 145 +} ;# foreach_detail_mode 142 146 143 147 finish_test 144 148
Changes to ext/fts5/test/fts5ag.test.
29 29 # ... WHERE fts MATCH ? ORDER BY bm25(fts) [ASC|DESC] 30 30 # 31 31 # and 32 32 # 33 33 # ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC] 34 34 # 35 35 36 +foreach_detail_mode $testprefix { 37 + 36 38 do_execsql_test 1.0 { 37 - CREATE VIRTUAL TABLE t1 USING fts5(x, y, z); 39 + CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%); 38 40 } 39 41 40 42 do_test 1.1 { 41 43 foreach {x y z} { 42 44 {j s m y m r n l u k} {z k f u z g h s w g} {r n o s s b v n w w} 43 45 {m v g n d x q r r s} {q t d a q a v l h j} {s k l f s i n v q v} 44 46 {m f f d h h s o h a} {y e v r q i u m h d} {b c k q m z l z h n} ................................................................................ 115 117 116 118 foreach {tn expr} { 117 119 2.1 a 118 120 2.2 b 119 121 2.3 c 120 122 2.4 d 121 123 122 - 2.5 {"m m"} 123 - 2.6 {e + s} 124 - 125 124 3.0 {a AND b} 126 125 3.1 {a OR b} 127 126 3.2 {b OR c AND d} 128 - 3.3 {NEAR(c d)} 129 127 } { 130 128 do_fts5ag_test $tn $expr 129 +} 131 130 132 - if {[set_test_counter errors]} break 131 +if {[detail_is_full]} { 132 + foreach {tn expr} { 133 + 4.1 {"m m"} 134 + 4.2 {e + s} 135 + 4.3 {NEAR(c d)} 136 + } { 137 + do_fts5ag_test $tn $expr 138 + } 133 139 } 134 140 141 +} ;# foreach_detail_mode 135 142 136 143 137 144 finish_test 138 145
Changes to ext/fts5/test/fts5ah.test.
16 16 set testprefix fts5ah 17 17 18 18 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 19 19 ifcapable !fts5 { 20 20 finish_test 21 21 return 22 22 } 23 + 24 +foreach_detail_mode $testprefix { 23 25 24 26 #------------------------------------------------------------------------- 25 27 # This file contains tests for very large doclists. 26 28 # 27 29 30 +set Y [list] 31 +set W [list] 28 32 do_test 1.0 { 29 - execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) } 33 + execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) } 30 34 execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } 31 35 set v {w w w w w w w w w w w w w w w w w w w w} 32 36 execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) } 33 37 for {set i 1} {$i <= 10000} {incr i} { 34 38 set v {x x x x x x x x x x x x x x x x x x x x} 35 39 if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i} 36 40 if {($i % 1577)==0} {lset v 5 W ; lappend W $i} ................................................................................ 66 70 expr [reads] - $nRead 67 71 } 68 72 69 73 do_test 1.4 { 70 74 set nRead [reads] 71 75 execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' } 72 76 set nReadX [expr [reads] - $nRead] 73 - expr $nReadX>1000 77 + #puts -nonewline "(nReadX=$nReadX)" 78 + if {[detail_is_full]} { set expect 1000 } 79 + if {[detail_is_col]} { set expect 250 } 80 + if {[detail_is_none]} { set expect 80 } 81 + 82 + expr $nReadX>$expect 74 83 } {1} 75 84 76 85 do_test 1.5 { 77 86 set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }] 78 87 set bwd [execsql_reads { 79 88 SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC 80 89 }] ................................................................................ 83 92 84 93 foreach {tn q res} " 85 94 1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W] 86 95 2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W] 87 96 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] 88 97 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] 89 98 " { 99 + if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue 100 + 101 + if {[detail_is_full]} { set ratio 8 } 102 + if {[detail_is_col]} { set ratio 4 } 103 + if {[detail_is_none]} { set ratio 2 } 90 104 91 105 do_test 1.6.$tn.1 { 92 106 set n [execsql_reads $q] 93 107 #puts -nonewline "(n=$n nReadX=$nReadX)" 94 - expr {$n < ($nReadX / 8)} 108 + expr {$n < ($nReadX / $ratio)} 95 109 } {1} 96 110 97 111 do_test 1.6.$tn.2 { 98 112 set n [execsql_reads "$q ORDER BY rowid DESC"] 99 113 #puts -nonewline "(n=$n nReadX=$nReadX)" 100 - expr {$n < ($nReadX / 8)} 114 + expr {$n < ($nReadX / $ratio)} 101 115 } {1} 102 116 103 117 do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res] 104 118 do_execsql_test 1.6.$tn.4 "$q ORDER BY rowid DESC" [lsort -int -decr $res] 105 119 } 106 120 107 121 #------------------------------------------------------------------------- 108 122 # Now test that adding range constraints on the rowid field reduces the 109 123 # number of pages loaded from disk. 110 124 # 111 125 foreach {tn fraction tail cnt} { 112 - 1 0.6 {rowid > 5000} 5000 113 - 2 0.2 {rowid > 9000} 1000 114 - 3 0.2 {rowid < 1000} 999 115 - 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 116 - 5 0.6 {rowid >= 5000} 5001 117 - 6 0.2 {rowid >= 9000} 1001 118 - 7 0.2 {rowid <= 1000} 1000 119 - 8 0.6 {rowid > '5000'} 5000 120 - 9 0.2 {rowid > '9000'} 1000 126 + 1 0.6 {rowid > 5000} 5000 127 + 2 0.2 {rowid > 9000} 1000 128 + 3 0.2 {rowid < 1000} 999 129 + 4 0.2 {rowid BETWEEN 4000 AND 5000} 1001 130 + 5 0.6 {rowid >= 5000} 5001 131 + 6 0.2 {rowid >= 9000} 1001 132 + 7 0.2 {rowid <= 1000} 1000 133 + 8 0.6 {rowid > '5000'} 5000 134 + 9 0.2 {rowid > '9000'} 1000 121 135 10 0.1 {rowid = 444} 1 122 136 } { 123 137 set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail" 124 138 set n [execsql_reads $q] 125 139 set ret [llength [execsql $q]] 126 140 141 + # Because the position lists for 'x' are quite long in this db, the 142 + # advantage is a bit smaller in detail=none mode. Update $fraction to 143 + # reflect this. 144 + if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] } 145 + 127 146 do_test "1.7.$tn.asc.(n=$n ret=$ret)" { 128 147 expr {$n < ($fraction*$nReadX) && $ret==$cnt} 129 148 } {1} 130 149 131 150 set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail ORDER BY rowid DESC" 132 151 set n [execsql_reads $q] 133 152 set ret [llength [execsql $q]] ................................................................................ 139 158 do_execsql_test 1.8.1 { 140 159 SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND +rowid < 'text'; 141 160 } {10000} 142 161 do_execsql_test 1.8.2 { 143 162 SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text'; 144 163 } {10000} 145 164 165 +} ;# foreach_detail_mode 146 166 147 167 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} 148 168 149 169 finish_test 150 170
Changes to ext/fts5/test/fts5ai.test.
19 19 20 20 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 21 21 ifcapable !fts5 { 22 22 finish_test 23 23 return 24 24 } 25 25 26 +foreach_detail_mode $testprefix { 27 + 26 28 do_execsql_test 1.0 { 27 - CREATE VIRTUAL TABLE t1 USING fts5(a); 29 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%); 28 30 } {} 29 31 30 32 do_execsql_test 1.1 { 31 33 BEGIN; 32 34 INSERT INTO t1 VALUES('a b c'); 33 35 INSERT INTO t1 VALUES('d e f'); 34 36 SAVEPOINT one; ................................................................................ 44 46 INSERT INTO t1 VALUES('s t u'); 45 47 ROLLBACK TO one; 46 48 COMMIT; 47 49 } 48 50 49 51 do_execsql_test 1.2 { 50 52 INSERT INTO t1(t1) VALUES('integrity-check'); 53 +} 51 54 } 52 55 53 56 54 57 finish_test 55 58
Changes to ext/fts5/test/fts5ak.test.
19 19 20 20 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 21 21 ifcapable !fts5 { 22 22 finish_test 23 23 return 24 24 } 25 25 26 +foreach_detail_mode $testprefix { 27 + 26 28 do_execsql_test 1.1 { 27 - CREATE VIRTUAL TABLE ft1 USING fts5(x); 29 + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); 28 30 INSERT INTO ft1 VALUES('i d d a g i b g d d'); 29 31 INSERT INTO ft1 VALUES('h d b j c c g a c a'); 30 32 INSERT INTO ft1 VALUES('e j a e f h b f h h'); 31 33 INSERT INTO ft1 VALUES('j f h d g h i b d f'); 32 34 INSERT INTO ft1 VALUES('d c j d c j b c g e'); 33 35 INSERT INTO ft1 VALUES('i a d e g j g d a a'); 34 36 INSERT INTO ft1 VALUES('j f c e d a h j d b'); 35 37 INSERT INTO ft1 VALUES('i c c f a d g h j e'); 36 38 INSERT INTO ft1 VALUES('i d i g c d c h b f'); 37 39 INSERT INTO ft1 VALUES('g d a e h a b c f j'); 40 + 41 + CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%); 42 + INSERT INTO ft2 VALUES('a b c d e f g h i j'); 38 43 } 39 44 40 45 do_execsql_test 1.2 { 41 46 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e'; 42 47 } { 43 48 {[e] j a [e] f h b f h h} 44 49 {d c j d c j b c g [e]} ................................................................................ 45 50 {i a d [e] g j g d a a} 46 51 {j f c [e] d a h j d b} 47 52 {i c c f a d g h j [e]} 48 53 {g d a [e] h a b c f j} 49 54 } 50 55 51 56 do_execsql_test 1.3 { 52 - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; 53 -} { 54 - {[h d] b j c c g a c a} 55 - {j f [h d] g h i b d f} 56 -} 57 - 58 -do_execsql_test 1.4 { 59 - SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; 60 -} { 61 - {i [d d] a g i b g [d d]} 62 -} 63 - 64 -do_execsql_test 1.5 { 65 57 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e' 66 58 } { 67 59 {[e] j a [e] f h b f h h} 68 60 {d c j d c j b c g [e]} 69 61 {i a d [e] g j g d a a} 70 62 {j f c [e] d a h j d b} 71 63 {i c c f a d g h j [e]} 72 64 {g d a [e] h a b c f j} 73 65 } 74 66 75 -do_execsql_test 1.6 { 67 +do_execsql_test 1.4 { 68 + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' 69 +} { 70 + {a b c [d] e [f] g h i j} 71 +} 72 + 73 +do_execsql_test 1.5 { 74 + SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' 75 +} { 76 + {a b c [d] e [f] g h i j} 77 +} 78 + 79 +#------------------------------------------------------------------------- 80 +# Tests below this point require detail=full. 81 +#------------------------------------------------------------------------- 82 +if {[detail_is_full]==0} continue 83 + 84 + 85 +do_execsql_test 2.1 { 86 + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d'; 87 +} { 88 + {[h d] b j c c g a c a} 89 + {j f [h d] g h i b d f} 90 +} 91 + 92 +do_execsql_test 2.2 { 93 + SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d'; 94 +} { 95 + {i [d d] a g i b g [d d]} 96 +} 97 + 98 +do_execsql_test 2.3 { 76 99 SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d'; 77 100 } { 78 101 {i [d d] a g i b g [d d]} 79 102 } 80 103 81 -do_execsql_test 2.1 { 82 - CREATE VIRTUAL TABLE ft2 USING fts5(x); 83 - INSERT INTO ft2 VALUES('a b c d e f g h i j'); 84 -} 85 - 86 -do_execsql_test 2.2 { 104 +do_execsql_test 2.4 { 87 105 SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e' 88 106 } {{a [b c d e] f g h i j}} 89 107 90 -do_execsql_test 2.3 { 108 +do_execsql_test 2.5 { 91 109 SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g' 92 110 } { 93 111 {a [b c d] [e f g] h i j} 94 112 } 95 113 96 -do_execsql_test 2.4 { 114 +do_execsql_test 2.6 { 97 115 SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c' 98 116 } { 99 117 {a [b c d] e f g h i j} 100 118 } 101 119 102 -do_execsql_test 2.5 { 120 +do_execsql_test 2.7 { 103 121 SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e' 104 122 } { 105 123 {a [b c d e] f g h i j} 106 124 } 107 125 108 -do_execsql_test 2.6.1 { 109 - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d' 110 -} { 111 - {a b c [d] e [f] g h i j} 112 -} 113 - 114 -do_execsql_test 2.6.2 { 115 - SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f' 116 -} { 117 - {a b c [d] e [f] g h i j} 118 -} 119 - 120 126 #------------------------------------------------------------------------- 121 127 # The example from the docs. 122 128 # 123 129 do_execsql_test 3.1 { 124 130 -- Assuming this: 125 - CREATE VIRTUAL TABLE ft USING fts5(a); 131 + CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%); 126 132 INSERT INTO ft VALUES('a b c x c d e'); 127 133 INSERT INTO ft VALUES('a b c c d e'); 128 134 INSERT INTO ft VALUES('a b c d e'); 129 135 130 136 -- The following SELECT statement returns these three rows: 131 137 -- '[a b c] x [c d e]' 132 138 -- '[a b c] [c d e]' ................................................................................ 134 140 SELECT highlight(ft, 0, '[', ']') FROM ft WHERE ft MATCH 'a+b+c AND c+d+e'; 135 141 } { 136 142 {[a b c] x [c d e]} 137 143 {[a b c] [c d e]} 138 144 {[a b c d e]} 139 145 } 140 146 147 +} 141 148 142 149 finish_test 143 150
Changes to ext/fts5/test/fts5al.test.
19 19 20 20 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 21 21 ifcapable !fts5 { 22 22 finish_test 23 23 return 24 24 } 25 25 26 +foreach_detail_mode $testprefix { 27 + 26 28 do_execsql_test 1.1 { 27 - CREATE VIRTUAL TABLE ft1 USING fts5(x); 29 + CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%); 28 30 SELECT * FROM ft1_config; 29 31 } {version 4} 30 32 31 33 do_execsql_test 1.2 { 32 34 INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); 33 35 SELECT * FROM ft1_config; 34 36 } {pgsz 32 version 4} ................................................................................ 79 81 } 80 82 81 83 #------------------------------------------------------------------------- 82 84 # Assorted tests of the tcl interface for creating extension functions. 83 85 # 84 86 85 87 do_execsql_test 3.1 { 86 - CREATE VIRTUAL TABLE t1 USING fts5(x); 88 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 87 89 INSERT INTO t1 VALUES('q w e r t y'); 88 90 INSERT INTO t1 VALUES('y t r e w q'); 89 91 } 90 92 91 93 proc argtest {cmd args} { return $args } 92 94 sqlite3_fts5_create_function db argtest argtest 93 95 ................................................................................ 118 120 do_execsql_test 3.4.1 { 119 121 SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'q' 120 122 } { 121 123 {{0 0 0}} 122 124 {{0 0 5}} 123 125 } 124 126 125 -do_execsql_test 3.4.2 { 126 - SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' 127 -} { 128 - {{1 0 1}} 129 - {{0 0 2} {1 0 4}} 127 +if {[detail_is_full]} { 128 + do_execsql_test 3.4.2 { 129 + SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w' 130 + } { 131 + {{1 0 1}} 132 + {{0 0 2} {1 0 4}} 133 + } 130 134 } 131 135 132 136 proc coltest {cmd} { 133 137 list [$cmd xColumnSize 0] [$cmd xColumnText 0] 134 138 } 135 139 sqlite3_fts5_create_function db coltest coltest 136 140 ................................................................................ 145 149 # Tests for remapping the "rank" column. 146 150 # 147 151 # 4.1.*: Mapped to a function with no arguments. 148 152 # 4.2.*: Mapped to a function with one or more arguments. 149 153 # 150 154 151 155 do_execsql_test 4.0 { 152 - CREATE VIRTUAL TABLE t2 USING fts5(a, b); 156 + CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%); 153 157 INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e'); 154 158 INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q'); 155 159 INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t'); 156 160 INSERT INTO t2 VALUES('h j g t r e l s g s', 'k q k c i i c k n s'); 157 161 INSERT INTO t2 VALUES('b l k h d n n n m i', 'p t i a r b t q o l'); 158 162 INSERT INTO t2 VALUES('k r i l j b g i p a', 't q c h a i m g n l'); 159 163 INSERT INTO t2 VALUES('a e c q n m o m d g', 'l c t g i s q g q e'); ................................................................................ 214 218 } 215 219 216 220 proc rowidplus {cmd ival} { 217 221 expr [$cmd xRowid] + $ival 218 222 } 219 223 sqlite3_fts5_create_function db rowidplus rowidplus 220 224 221 -do_execsql_test 4.2.1 { 222 - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); 223 - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' 224 -} { 225 - 10 110 226 -} 227 -do_execsql_test 4.2.2 { 228 - INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); 229 - SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' 230 -} { 231 - 10 121 232 -} 225 +if {[detail_is_full]} { 226 + do_execsql_test 4.2.1 { 227 + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) '); 228 + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' 229 + } { 230 + 10 110 231 + } 232 + do_execsql_test 4.2.2 { 233 + INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) '); 234 + SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g' 235 + } { 236 + 10 121 237 + } 233 238 234 -do_execsql_test 4.2.3 { 235 - SELECT rowid, rank FROM t2 236 - WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' 237 -} { 238 - 10 122 239 + do_execsql_test 4.2.3 { 240 + SELECT rowid, rank FROM t2 241 + WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)' 242 + } { 243 + 10 122 244 + } 239 245 } 240 246 241 247 proc rowidmod {cmd imod} { 242 248 expr [$cmd xRowid] % $imod 243 249 } 244 250 sqlite3_fts5_create_function db rowidmod rowidmod 245 251 do_execsql_test 4.3.1 { 246 - CREATE VIRTUAL TABLE t3 USING fts5(x); 252 + CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%); 247 253 INSERT INTO t3 VALUES('a one'); 248 254 INSERT INTO t3 VALUES('a two'); 249 255 INSERT INTO t3 VALUES('a three'); 250 256 INSERT INTO t3 VALUES('a four'); 251 257 INSERT INTO t3 VALUES('a five'); 252 258 INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()'); 253 259 } ................................................................................ 283 289 do_catchsql_test 4.4.3 { 284 290 SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 285 291 } {1 {no such function: xyz}} 286 292 do_catchsql_test 4.4.4 { 287 293 SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL 288 294 } {1 {parse error in rank function: }} 289 295 296 +} ;# foreach_detail_mode 290 297 291 298 292 299 finish_test 293 300
Changes to ext/fts5/test/fts5auto.test.
18 18 19 19 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 20 20 ifcapable !fts5 { 21 21 finish_test 22 22 return 23 23 } 24 24 25 - 26 25 set data { 27 26 -4026076 28 27 {n x w k b p x b n t t d s} {f j j s p j o} 29 28 {w v i y r} {i p y s} 30 29 {a o q v e n q r} {q v g u c y a z y} 31 30 3995120 32 31 {c} {e e w d t} ................................................................................ 228 227 229 228 do_execsql_test 1.0 { 230 229 CREATE VIRTUAL TABLE tt USING fts5(a, b, c, d, e, f); 231 230 } {} 232 231 233 232 fts5_aux_test_functions db 234 233 235 -proc matchdata {expr tbl collist {order ASC}} { 236 - 237 - set cols "" 238 - foreach e $collist { 239 - append cols ", '$e'" 240 - } 241 - 242 - set tclexpr [db one [subst -novar { 243 - SELECT fts5_expr_tcl( 244 - $expr, 'nearset $cols -pc ::pc' [set cols] 245 - ) 246 - }]] 247 - set res [list] 248 - 249 - db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x { 250 - set cols [list] 251 - foreach col $x(*) { 252 - if {$col != "rowid"} { lappend cols $x($col) } 253 - } 254 - # set cols [list $a $b $c $d $e $f] 255 - set ::pc 0 256 - set rowdata [eval $tclexpr] 257 - if {$rowdata != ""} { lappend res $x(rowid) $rowdata } 258 - } 259 - 260 - set res 261 -} 262 - 263 -proc do_auto_test {tn tbl cols expr} { 234 +proc do_auto_test {tn tbl expr} { 264 235 foreach order {asc desc} { 265 - set res [matchdata $expr $tbl $cols $order] 236 + set res [fts5_poslist_data $expr $tbl $order] 266 237 set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]" 267 238 268 239 set ::autotest_expr $expr 269 240 do_execsql_test $testname [subst -novar { 270 241 SELECT rowid, fts5_test_poslist([set tbl]) FROM [set tbl] 271 242 WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order] 272 243 }] $res 273 244 } 274 - 275 - 276 245 } 277 246 278 247 #------------------------------------------------------------------------- 279 248 # 280 249 281 250 for {set fold 0} {$fold < 3} {incr fold} { 282 251 switch $fold { ................................................................................ 328 297 B.4 { a OR (b AND {a b c}:c) } 329 298 B.5 { a OR "b c" } 330 299 B.6 { a OR b OR c } 331 300 332 301 C.1 { a OR (b AND "b c") } 333 302 C.2 { a OR (b AND "z c") } 334 303 } { 335 - do_auto_test 3.$fold.$tn tt {a b c d e f} $expr 304 + do_auto_test 3.$fold.$tn tt $expr 336 305 } 337 306 } 338 307 339 308 proc replace_elems {list args} { 340 309 set ret $list 341 310 foreach {idx elem} $args { 342 311 set ret [lreplace $ret $idx $idx $elem] ................................................................................ 362 331 1 x 363 332 2 y 364 333 3 z 365 334 366 335 4 {c1 : x} 5 {c2 : x} 6 {c3 : x} 367 336 7 {c1 : y} 8 {c2 : y} 9 {c3 : y} 368 337 10 {c1 : z} 11 {c2 : z} 12 {c3 : z} 369 - 370 - 371 338 } { 372 -breakpoint 373 - do_auto_test 4.$tn yy {c1 c2 c3} $expr 339 + do_auto_test 4.$tn yy $expr 374 340 } 375 341 376 342 377 343 378 344 finish_test 379 345
Added ext/fts5/test/fts5detail.test.
1 +# 2015 December 18 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# This file implements regression tests for SQLite library. The 12 +# focus of this script is testing the FTS5 module. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5detail 17 + 18 +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +fts5_aux_test_functions db 25 + 26 +#-------------------------------------------------------------------------- 27 +# Simple tests. 28 +# 29 +do_execsql_test 1.0 { 30 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); 31 + INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 32 + INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 33 + INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 34 + INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 35 + INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 36 + INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 37 + INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 38 + INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8 39 + INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9 40 + INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10 41 +} 42 + 43 +do_execsql_test 1.1 { 44 + INSERT INTO t1(t1) VALUES('integrity-check'); 45 +} 46 + 47 +foreach {tn match res} { 48 + 1 "a:a" {9} 49 + 2 "b:g" {1 4 6} 50 + 3 "c:h" {1 3 6 8 9 10} 51 +} { 52 + do_execsql_test 1.2.$tn.1 { 53 + SELECT rowid FROM t1($match); 54 + } $res 55 + 56 + do_execsql_test 1.2.$tn.2 { 57 + SELECT rowid FROM t1($match || '*'); 58 + } $res 59 +} 60 + 61 +do_catchsql_test 1.3.1 { 62 + SELECT rowid FROM t1('h + d'); 63 +} {1 {fts5: phrase queries are not supported (detail!=full)}} 64 + 65 +do_catchsql_test 1.3.2 { 66 + SELECT rowid FROM t1('NEAR(h d)'); 67 +} {1 {fts5: NEAR queries are not supported (detail!=full)}} 68 + 69 + 70 +#------------------------------------------------------------------------- 71 +# integrity-check with both detail= and prefix= options. 72 +# 73 +do_execsql_test 2.0 { 74 + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); 75 + INSERT INTO t2(a) VALUES('aa ab'); 76 +} 77 + 78 +#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} 79 + 80 +do_execsql_test 2.1 { 81 + INSERT INTO t2(t2) VALUES('integrity-check'); 82 +} 83 + 84 +do_execsql_test 2.2 { 85 + SELECT fts5_test_poslist(t2) FROM t2('aa'); 86 +} {0.0.0} 87 + 88 +set ::pc 0 89 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] 90 +#exit 91 + 92 +#------------------------------------------------------------------------- 93 +# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs 94 +# work with detail=col tables. 95 +# 96 +set data { 97 + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} 98 + 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} 99 + 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} 100 + 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} 101 + 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} 102 + 6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb} 103 + 7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc} 104 + 8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb} 105 + 9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb} 106 + 10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa} 107 + 11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac} 108 + 12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc} 109 + 13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc} 110 + 14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa} 111 + 15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc} 112 + 16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc} 113 + 17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca} 114 + 18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc} 115 + 19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc} 116 + 20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc} 117 + 21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb} 118 + 22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc} 119 + 23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc} 120 + 24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb} 121 + 25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc} 122 + 26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa} 123 + 27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac} 124 + 28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa} 125 + 29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac} 126 + 30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac} 127 + 31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba} 128 + 32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc} 129 + 33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa} 130 + 34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac} 131 + 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} 132 + 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} 133 + 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} 134 + 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} 135 + 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} 136 + 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} 137 +} 138 + 139 +set data { 140 + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} 141 +} 142 + 143 +proc matchdata {expr {bAsc 1}} { 144 + 145 + set tclexpr [db one { 146 + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') 147 + }] 148 + set res [list] 149 + 150 + #puts "$expr -> $tclexpr" 151 + foreach {id x y z} $::data { 152 + set cols [list $x $y $z] 153 + set ::pc 0 154 + #set hits [lsort -command instcompare [eval $tclexpr]] 155 + set hits [eval $tclexpr] 156 + if {[llength $hits]>0} { 157 + lappend res [list $id $hits] 158 + } 159 + } 160 + 161 + if {$bAsc} { 162 + set res [lsort -integer -increasing -index 0 $res] 163 + } else { 164 + set res [lsort -integer -decreasing -index 0 $res] 165 + } 166 + 167 + return [concat {*}$res] 168 +} 169 + 170 +foreach {tn tbl} { 171 + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } 172 + 2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) } 173 +} { 174 + reset_db 175 + fts5_aux_test_functions db 176 + execsql $tbl 177 + foreach {id x y z} $data { 178 + execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } 179 + } 180 + foreach {tn2 expr} { 181 + 1 aaa 2 ccc 3 bab 4 aac 182 + 5 aa* 6 cc* 7 ba* 8 aa* 183 + 9 a* 10 b* 11 c* 184 + } { 185 + 186 + set res [matchdata $expr] 187 + 188 + do_execsql_test 3.$tn.$tn2.1 { 189 + SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) 190 + } $res 191 + 192 + do_execsql_test 3.$tn.$tn2.2 { 193 + SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) 194 + } $res 195 + } 196 +} 197 + 198 +#------------------------------------------------------------------------- 199 +# Simple tests for detail=none tables. 200 +# 201 +do_execsql_test 4.0 { 202 + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); 203 + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); 204 + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); 205 +} 206 + 207 +do_catchsql_test 4.1 { 208 + SELECT * FROM t4('a:a') 209 +} {1 {fts5: column queries are not supported (detail=none)}} 210 + 211 +#------------------------------------------------------------------------- 212 +# Test that for the same content detail=none uses less space than 213 +# detail=col, and that detail=col uses less space than detail=full 214 +# 215 +reset_db 216 +do_test 5.1 { 217 + foreach {tbl detail} {t1 none t2 col t3 full} { 218 + execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)" 219 + foreach {rowid x y z} $::data { 220 + execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)" 221 + } 222 + } 223 +} {} 224 + 225 +do_execsql_test 5.2 { 226 + SELECT 227 + (SELECT sum(length(block)) from t1_data) < 228 + (SELECT sum(length(block)) from t2_data) 229 +} {1} 230 + 231 +do_execsql_test 5.3 { 232 + SELECT 233 + (SELECT sum(length(block)) from t2_data) < 234 + (SELECT sum(length(block)) from t3_data) 235 +} {1} 236 + 237 + 238 + 239 +finish_test 240 +
Changes to ext/fts5/test/fts5dlidx.test.
22 22 } 23 23 24 24 if { $tcl_platform(wordSize)<8 } { 25 25 finish_test 26 26 return 27 27 } 28 28 29 -if 1 { 29 +foreach_detail_mode $testprefix { 30 30 31 31 proc do_fb_test {tn sql res} { 32 32 set res2 [lsort -integer -decr $res] 33 33 uplevel [list do_execsql_test $tn.1 $sql $res] 34 34 uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2] 35 35 } 36 36 37 -# This test populates the FTS5 table containing $nEntry entries. Rows are 37 +# This test populates the FTS5 table with $nEntry entries. Rows are 38 38 # numbered from 0 to ($nEntry-1). The rowid for row $i is: 39 39 # 40 40 # ($iFirst + $i*$nStep) 41 41 # 42 42 # Each document is of the form "a b c a b c a b c...". If the row number ($i) 43 43 # is an integer multiple of $spc1, then an "x" token is appended to the 44 44 # document. If it is *also* a multiple of $spc2, a "y" token is also appended. ................................................................................ 73 73 74 74 do_fb_test $tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc 75 75 do_fb_test $tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc 76 76 77 77 do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc 78 78 do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc 79 79 80 - do_fb_test $tn.5.1 { 81 - SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc 82 - do_fb_test $tn.5.2 { 83 - SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc 80 + if {[detail_is_full]} { 81 + do_fb_test $tn.5.1 { 82 + SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc 83 + do_fb_test $tn.5.2 { 84 + SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc 85 + } 84 86 } 85 87 86 88 87 89 foreach {tn pgsz} { 88 90 1 32 89 91 2 200 90 92 } { 91 93 do_execsql_test $tn.0 { 92 94 DROP TABLE IF EXISTS t1; 93 - CREATE VIRTUAL TABLE t1 USING fts5(x); 95 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 94 96 INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); 95 97 } 96 98 97 99 do_dlidx_test1 1.$tn.1 10 100 10000 0 1000 98 100 do_dlidx_test1 1.$tn.2 10 10 10000 0 128 99 101 do_dlidx_test1 1.$tn.3 10 10 66 0 36028797018963970 100 102 do_dlidx_test1 1.$tn.4 10 10 50 0 150000000000000000 ................................................................................ 103 105 } 104 106 105 107 proc do_dlidx_test2 {tn nEntry iFirst nStep} { 106 108 set str [string repeat "a " 500] 107 109 execsql { 108 110 BEGIN; 109 111 DROP TABLE IF EXISTS t1; 110 - CREATE VIRTUAL TABLE t1 USING fts5(x); 112 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 111 113 INSERT INTO t1(t1, rank) VALUES('pgsz', 64); 112 114 INSERT INTO t1 VALUES('b a'); 113 115 114 116 WITH iii(ii, i) AS ( 115 117 SELECT 1, $iFirst UNION ALL 116 118 SELECT ii+1, i+$nStep FROM iii WHERE ii<$nEntry 117 119 ) ................................................................................ 126 128 do_execsql_test $tn.2 { 127 129 SELECT rowid FROM t1 WHERE t1 MATCH 'b AND a' ORDER BY rowid DESC 128 130 } {1} 129 131 } 130 132 131 133 do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128] 132 134 133 -} 134 - 135 135 #-------------------------------------------------------------------- 136 136 # 137 137 reset_db 138 138 139 139 set ::vocab [list \ 140 140 IteratorpItercurrentlypointstothefirstrowidofadoclist \ 141 141 Thereisadoclistindexassociatedwiththefinaltermonthecurrent \ ................................................................................ 154 154 lappend ret [lindex $vocab [expr $i % $nVocab]] 155 155 } 156 156 set ret 157 157 } 158 158 db func rnddoc rnddoc 159 159 160 160 do_execsql_test 3.1 { 161 - CREATE VIRTUAL TABLE abc USING fts5(a); 161 + CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%); 162 162 INSERT INTO abc(abc, rank) VALUES('pgsz', 32); 163 163 164 164 INSERT INTO abc VALUES ( rnddoc() ); 165 165 INSERT INTO abc VALUES ( rnddoc() ); 166 166 INSERT INTO abc VALUES ( rnddoc() ); 167 167 INSERT INTO abc VALUES ( rnddoc() ); 168 168 ................................................................................ 187 187 set v [lindex $vocab 0] 188 188 set i 0 189 189 foreach v $vocab { 190 190 do_execsql_test 3.3.[incr i] { 191 191 SELECT rowid FROM abc WHERE abc MATCH $v 192 192 } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} 193 193 } 194 + 195 +} ;# foreach_detail_mode 196 + 194 197 195 198 196 199 finish_test 197 200
Changes to ext/fts5/test/fts5eb.test.
38 38 6 {abc AND ""} {"abc"} 39 39 7 {"" OR abc} {"abc"} 40 40 8 {"" NOT abc} {"abc"} 41 41 9 {"" AND abc} {"abc"} 42 42 10 {abc + "" + def} {"abc" + "def"} 43 43 11 {abc "" def} {"abc" AND "def"} 44 44 12 {r+e OR w} {"r" + "e" OR "w"} 45 + 46 + 13 {a AND b NOT c} {"a" AND ("b" NOT "c")} 47 + 14 {a OR b NOT c} {"a" OR ("b" NOT "c")} 48 + 15 {a NOT b AND c} {("a" NOT "b") AND "c"} 49 + 16 {a NOT b OR c} {("a" NOT "b") OR "c"} 50 + 51 + 17 {a AND b OR c} {("a" AND "b") OR "c"} 52 + 18 {a OR b AND c} {"a" OR ("b" AND "c")} 53 + 45 54 } { 46 55 do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res] 47 56 } 48 57 49 58 do_catchsql_test 2.1 { 50 59 SELECT fts5_expr() 51 60 } {1 {wrong number of arguments to function fts5_expr}}
Added ext/fts5/test/fts5fault8.test.
1 +# 2015 September 3 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# 12 +# This file is focused on OOM errors. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +source $testdir/malloc_common.tcl 17 +set testprefix fts5fault8 18 + 19 +# If SQLITE_ENABLE_FTS3 is defined, omit this file. 20 +ifcapable !fts5 { 21 + finish_test 22 + return 23 +} 24 + 25 +foreach_detail_mode $testprefix { 26 + 27 +if {[detail_is_none]==0} continue 28 + 29 +fts5_aux_test_functions db 30 +do_execsql_test 1.0 { 31 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%); 32 + INSERT INTO t1 VALUES('a b c d', '1 2 3 4'); 33 + INSERT INTO t1 VALUES('a b a b', NULL); 34 + INSERT INTO t1 VALUES(NULL, '1 2 1 2'); 35 +} 36 + 37 +do_faultsim_test 1 -faults oom-t* -body { 38 + execsql { 39 + SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2' 40 + } 41 +} -test { 42 + faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \ 43 + {1 SQLITE_NOMEM} 44 +} 45 + 46 +} 47 + 48 +finish_test 49 +
Changes to ext/fts5/test/fts5matchinfo.test.
12 12 13 13 source [file join [file dirname [info script]] fts5_common.tcl] 14 14 set testprefix fts5matchinfo 15 15 16 16 # If SQLITE_ENABLE_FTS5 is not defined, omit this file. 17 17 ifcapable !fts5 { finish_test ; return } 18 18 19 +foreach_detail_mode $testprefix { 20 + 19 21 proc mit {blob} { 20 22 set scan(littleEndian) i* 21 23 set scan(bigEndian) I* 22 24 binary scan $blob $scan($::tcl_platform(byteOrder)) r 23 25 return $r 24 26 } 25 27 db func mit mit 26 28 27 29 sqlite3_fts5_register_matchinfo db 28 30 29 31 do_execsql_test 1.0 { 30 - CREATE VIRTUAL TABLE t1 USING fts5(content); 32 + CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%); 31 33 } 32 34 33 35 do_execsql_test 1.1 { 34 36 INSERT INTO t1(content) VALUES('I wandered lonely as a cloud'); 35 37 INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,'); 36 38 INSERT INTO t1(content) VALUES('When all at once I saw a crowd,'); 37 39 INSERT INTO t1(content) VALUES('A host, of golden daffodils,'); 38 40 SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I'; 39 41 } {{1 1 1 2 2} {1 1 1 2 2}} 40 42 41 43 # Now create an FTS4 table that does not specify matchinfo=fts3. 42 44 # 43 45 do_execsql_test 1.2 { 44 - CREATE VIRTUAL TABLE t2 USING fts5(content); 46 + CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%); 45 47 INSERT INTO t2 SELECT * FROM t1; 46 48 SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I'; 47 49 } {{1 1 1 2 2} {1 1 1 2 2}} 48 50 49 51 50 52 #-------------------------------------------------------------------------- 51 53 # Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function. ................................................................................ 145 147 set res [list] 146 148 foreach elem $list_of_lists { 147 149 lappend res [list {*}$elem] 148 150 } 149 151 return $res 150 152 } 151 153 154 +# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5 155 +# mode is not detail=full. 156 +# 157 +proc do_matchinfo_p_test {tn tbl expr results} { 158 + if {[detail_is_full]} { 159 + uplevel [list do_matchinfo_test $tn $tbl $expr $results] 160 + } 161 +} 152 162 153 163 do_execsql_test 4.1.0 { 154 - CREATE VIRTUAL TABLE t4 USING fts5(x, y); 164 + CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%); 155 165 INSERT INTO t4 VALUES('a b c d e', 'f g h i j'); 156 166 INSERT INTO t4 VALUES('f g h i j', 'a b c d e'); 157 167 } 158 168 159 169 do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} { 160 170 s {{3 0} {0 3}} 161 171 } ................................................................................ 181 191 182 192 s {{3 0} {0 3}} 183 193 184 194 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - 185 195 xpxsscplax - 186 196 } 187 197 188 -do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} { 198 +do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} { 189 199 p {1 1} 190 200 c {2 2} 191 201 x { 192 202 {0 1 1 1 1 1} 193 203 {1 1 1 0 1 1} 194 204 } 195 205 n {2 2} ................................................................................ 199 209 s {{0 1} {1 0}} 200 210 201 211 xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc - 202 212 sxsxs - 203 213 } 204 214 205 215 do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} } 206 -do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } 207 -do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } 216 +do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} } 217 +do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} } 208 218 do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} } 209 219 do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} { 210 220 x { 211 221 {0 1 1 1 1 1 0 0 0 0 0 0} 212 222 {1 1 1 0 1 1 0 0 0 0 0 0} 213 223 } 214 224 } ................................................................................ 216 226 x { 217 227 {0 1 1 1 1 1 0 0 0 0 0 0} 218 228 {1 1 1 0 1 1 0 0 0 0 0 0} 219 229 } 220 230 } 221 231 222 232 do_execsql_test 4.2.0 { 223 - CREATE VIRTUAL TABLE t5 USING fts5(content); 233 + CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%); 224 234 INSERT INTO t5 VALUES('a a a a a'); 225 235 INSERT INTO t5 VALUES('a b a b a'); 226 236 INSERT INTO t5 VALUES('c b c b c'); 227 237 INSERT INTO t5 VALUES('x x x x x'); 228 238 } 229 239 do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} { 230 240 x {{5 8 2 5 8 2} {3 8 2 3 8 2}} 231 241 s {2 1} 232 242 } 233 243 do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} } 234 244 do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} } 235 245 do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} } 236 -do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 246 +do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 237 247 do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } 238 248 239 249 do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; 240 250 241 251 # It used to be that the second 'a' token would be deferred. That doesn't 242 252 # work any longer. 243 253 if 0 { ................................................................................ 246 256 s {2 1} 247 257 } 248 258 } 249 259 250 260 do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } 251 261 do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } 252 262 do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } 253 -do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 263 +do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 254 264 do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } 255 265 256 266 do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } 257 267 258 268 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } 259 269 do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } 260 270 do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } 261 271 do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} } 262 272 do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} } 263 -do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 273 +do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } 264 274 265 275 do_execsql_test 4.5.0 { 266 - CREATE VIRTUAL TABLE t6 USING fts5(a, b, c); 276 + CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%); 267 277 INSERT INTO t6 VALUES('a', 'b', 'c'); 268 278 } 269 279 do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} } 270 280 271 281 272 282 #------------------------------------------------------------------------- 273 283 # Test the outcome of matchinfo() when used within a query that does not 274 284 # use the full-text index (i.e. lookup by rowid or full-table scan). 275 285 # 276 286 do_execsql_test 7.1 { 277 - CREATE VIRTUAL TABLE t10 USING fts5(content); 287 + CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%); 278 288 INSERT INTO t10 VALUES('first record'); 279 289 INSERT INTO t10 VALUES('second record'); 280 290 } 281 291 do_execsql_test 7.2 { 282 292 SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10; 283 293 } {blob 8 blob 8} 284 294 do_execsql_test 7.3 { ................................................................................ 295 305 # This was causing a problem at one point in the obscure case where the 296 306 # total number of bytes of data stored in an fts3 table was greater than 297 307 # the number of rows. i.e. when the following query returns true: 298 308 # 299 309 # SELECT sum(length(content)) < count(*) FROM fts4table; 300 310 # 301 311 do_execsql_test 8.1 { 302 - CREATE VIRTUAL TABLE t11 USING fts5(content); 312 + CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%); 303 313 INSERT INTO t11(t11, rank) VALUES('pgsz', 32); 304 314 INSERT INTO t11 VALUES('quitealongstringoftext'); 305 315 INSERT INTO t11 VALUES('anotherquitealongstringoftext'); 306 316 INSERT INTO t11 VALUES('athirdlongstringoftext'); 307 317 INSERT INTO t11 VALUES('andonemoreforgoodluck'); 308 318 } 309 319 do_test 8.2 { ................................................................................ 314 324 } {} 315 325 do_execsql_test 8.3 { 316 326 SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*' 317 327 } {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}} 318 328 319 329 #------------------------------------------------------------------------- 320 330 321 -do_execsql_test 9.1 { 322 - CREATE VIRTUAL TABLE t12 USING fts5(content); 323 - INSERT INTO t12 VALUES('a b c d'); 324 - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 325 -} {{0 1 1 0 1 1 1 1 1}} 326 -do_execsql_test 9.2 { 327 - INSERT INTO t12 VALUES('a d c d'); 328 - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 329 -} { 330 - {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} 331 -} 332 -do_execsql_test 9.3 { 333 - INSERT INTO t12 VALUES('a d d a'); 334 - SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 335 -} { 336 - {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} 331 +if {[detail_is_full]} { 332 + do_execsql_test 9.1 { 333 + CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%); 334 + INSERT INTO t12 VALUES('a b c d'); 335 + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 336 + } {{0 1 1 0 1 1 1 1 1}} 337 + do_execsql_test 9.2 { 338 + INSERT INTO t12 VALUES('a d c d'); 339 + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 340 + } { 341 + {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2} 342 + } 343 + do_execsql_test 9.3 { 344 + INSERT INTO t12 VALUES('a d d a'); 345 + SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a'; 346 + } { 347 + {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3} 348 + } 337 349 } 338 350 339 351 #--------------------------------------------------------------------------- 340 352 # Test for a memory leak 341 353 # 342 354 do_execsql_test 10.1 { 343 355 DROP TABLE t10; 344 - CREATE VIRTUAL TABLE t10 USING fts5(idx, value); 356 + CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%); 345 357 INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); 346 358 SELECT t10.rowid, t10.* 347 359 FROM t10 348 360 JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x 349 361 WHERE t10 MATCH x.idx 350 362 AND matchinfo(t10) not null 351 363 GROUP BY t10.rowid ................................................................................ 354 366 355 367 #--------------------------------------------------------------------------- 356 368 # Test the 'y' matchinfo flag 357 369 # 358 370 reset_db 359 371 sqlite3_fts5_register_matchinfo db 360 372 do_execsql_test 11.0 { 361 - CREATE VIRTUAL TABLE tt USING fts5(x, y); 373 + CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%); 362 374 INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1 363 375 INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2 364 376 INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3 365 377 INSERT INTO tt VALUES('a c f f g d', 'd b f d e g'); -- 4 366 378 INSERT INTO tt VALUES('g a c f c f', 'd g g b c c'); -- 5 367 379 INSERT INTO tt VALUES('g a c e b b', 'd b f b g g'); -- 6 368 380 INSERT INTO tt VALUES('f d a a f c', 'e e a d c f'); -- 7 ................................................................................ 406 418 7 "a OR (a AND b)" { 407 419 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} 408 420 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} 409 421 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} 410 422 } 411 423 412 424 } { 425 + 426 + if {[string match *:* $expr] && [detail_is_none]} continue 413 427 do_execsql_test 11.1.$tn.1 { 414 428 SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr 415 429 } $res 416 430 417 431 set r2 [list] 418 432 foreach {rowid L} $res { 419 433 lappend r2 $rowid ................................................................................ 439 453 reset_db 440 454 sqlite3_fts5_register_matchinfo db 441 455 db func mit mit 442 456 443 457 do_test 12.0 { 444 458 set cols [list] 445 459 for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } 446 - execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])" 460 + execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)" 447 461 } {} 448 462 449 463 do_execsql_test 12.1 { 450 464 INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); 451 465 SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; 452 466 } [list [list [expr 1<<4] [expr 1<<(45-32)]]] 467 + 468 +} ;# foreach_detail_mode 453 469 454 470 finish_test 455 471
Changes to ext/fts5/test/fts5simple.test.
14 14 set testprefix fts5simple 15 15 16 16 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 17 17 ifcapable !fts5 { 18 18 finish_test 19 19 return 20 20 } 21 - 21 + 22 + if 1 { 23 + 22 24 #------------------------------------------------------------------------- 23 25 # 24 26 set doc "x x [string repeat {y } 50]z z" 25 27 do_execsql_test 1.0 { 26 28 CREATE VIRTUAL TABLE t1 USING fts5(x); 27 29 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 28 30 BEGIN; ................................................................................ 346 348 INSERT INTO x1 SELECT rnddoc(5) FROM ii; 347 349 } 348 350 349 351 do_execsql_test 4.1 { 350 352 SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads' 351 353 } {0 {} 4} 352 354 355 +#------------------------------------------------------------------------- 356 +reset_db 357 +do_execsql_test 15.0 { 358 + CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1); 359 + INSERT INTO x2 VALUES('ab'); 360 +} 361 + 362 +do_execsql_test 15.1 { 363 + INSERT INTO x2(x2) VALUES('integrity-check'); 364 +} 365 + 366 +} 367 + 368 +#------------------------------------------------------------------------- 369 +foreach_detail_mode $testprefix { 370 + reset_db 371 + fts5_aux_test_functions db 372 + do_execsql_test 16.0 { 373 + CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%); 374 + INSERT INTO x3 VALUES('a b c d e f'); 375 + } 376 + do_execsql_test 16.1 { 377 + SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c'); 378 + } {2.0.2} 379 + 380 + do_execsql_test 16.1 { 381 + SELECT fts5_test_poslist(x3) FROM x3('a OR c'); 382 + } {{0.0.0 1.0.2}} 383 +} 353 384 354 385 finish_test 355 386
Added ext/fts5/test/fts5simple2.test.
1 +# 2015 September 05 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# 12 + 13 +source [file join [file dirname [info script]] fts5_common.tcl] 14 +set testprefix fts5simple2 15 + 16 +# If SQLITE_ENABLE_FTS5 is defined, omit this file. 17 +ifcapable !fts5 { 18 + finish_test 19 + return 20 +} 21 + 22 +if 1 { 23 + 24 +do_execsql_test 1.0 { 25 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 26 + INSERT INTO t1 VALUES('a b c'); 27 +} 28 +do_execsql_test 1.1 { 29 + SELECT rowid FROM t1('c a b') 30 +} {1} 31 + 32 +#------------------------------------------------------------------------- 33 +# 34 +reset_db 35 +do_execsql_test 2.0 { 36 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 37 + BEGIN; 38 + INSERT INTO t1 VALUES('b c d'); 39 + INSERT INTO t1 VALUES('b c d'); 40 + COMMIT; 41 +} 42 +do_execsql_test 2.1 { 43 + SELECT rowid FROM t1('b c d') 44 +} {1 2} 45 + 46 +#------------------------------------------------------------------------- 47 +# 48 +reset_db 49 +do_execsql_test 3.0 { 50 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 51 + BEGIN; 52 + INSERT INTO t1 VALUES('b c d'); 53 + INSERT INTO t1 VALUES('b c d'); 54 +} 55 +do_execsql_test 3.1 { 56 + SELECT rowid FROM t1('b c d'); COMMIT; 57 +} {1 2} 58 + 59 +#------------------------------------------------------------------------- 60 +# 61 +reset_db 62 +do_execsql_test 4.0 { 63 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 64 + BEGIN; 65 + INSERT INTO t1 VALUES('a1 b1 c1'); 66 + INSERT INTO t1 VALUES('a2 b2 c2'); 67 + INSERT INTO t1 VALUES('a3 b3 c3'); 68 + COMMIT; 69 +} 70 +do_execsql_test 4.1 { 71 + SELECT rowid FROM t1('b*'); 72 +} {1 2 3} 73 + 74 + 75 +#------------------------------------------------------------------------- 76 +# 77 +reset_db 78 +do_execsql_test 5.0 { 79 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 80 + BEGIN; 81 + INSERT INTO t1 VALUES('a1 b1 c1'); 82 + INSERT INTO t1 VALUES('a2 b2 c2'); 83 + INSERT INTO t1 VALUES('a1 b1 c1'); 84 + COMMIT; 85 +} 86 +do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3} 87 + 88 +#------------------------------------------------------------------------- 89 +# 90 +reset_db 91 +do_execsql_test 6.0 { 92 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full); 93 + BEGIN; 94 + INSERT INTO t1 VALUES('a1 b1 c1'); 95 + INSERT INTO t1 VALUES('a1 b1 c1'); 96 + INSERT INTO t1 VALUES('a1 b1 c1'); 97 + COMMIT; 98 +} 99 + 100 +do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1} 101 +do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1} 102 +do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1} 103 + 104 +#------------------------------------------------------------------------- 105 +# 106 +reset_db 107 +do_execsql_test 7.0 { 108 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 109 + BEGIN; 110 + INSERT INTO t1 VALUES('a1 b1'); 111 + INSERT INTO t1 VALUES('a1 b2'); 112 + COMMIT; 113 +} 114 +do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1} 115 +do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1} 116 + 117 +#------------------------------------------------------------------------- 118 +# 119 +reset_db 120 +do_execsql_test 8.0 { 121 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 122 + INSERT INTO t1 VALUES('a1 b1 c1'); 123 + INSERT INTO t1 VALUES('a2 b2 c2'); 124 + INSERT INTO t1 VALUES('a1 b1 c1'); 125 +} 126 +do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3} 127 +do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3} 128 +do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2} 129 + 130 +do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1} 131 +do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1} 132 +do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2} 133 + 134 +do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') } 135 + 136 +do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3} 137 +do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3} 138 +do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2} 139 + 140 +do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1} 141 +do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1} 142 +do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2} 143 + 144 +#-------------------------------------------------------------------------- 145 +# 146 +reset_db 147 +do_execsql_test 9.0.0 { 148 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 149 + INSERT INTO t1 VALUES('a1 b1 c1'); 150 + INSERT INTO t1 VALUES('a2 b2 c2'); 151 + INSERT INTO t1 VALUES('a1 b1 c1'); 152 +} 153 +do_execsql_test 9.0.1 { 154 + INSERT INTO t1(t1) VALUES('integrity-check'); 155 +} {} 156 + 157 +reset_db 158 +do_execsql_test 9.1.0 { 159 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none); 160 + INSERT INTO t1 VALUES('a1 b1 c1', 'x y z'); 161 + INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3'); 162 + INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z'); 163 +} 164 +do_execsql_test 9.2.1 { 165 + INSERT INTO t1(t1) VALUES('integrity-check'); 166 +} {} 167 + 168 +#-------------------------------------------------------------------------- 169 +# 170 +reset_db 171 +do_execsql_test 10.0 { 172 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 173 + INSERT INTO t1 VALUES('b1'); 174 + INSERT INTO t1 VALUES('b1'); 175 + DELETE FROM t1 WHERE rowid=1; 176 +} 177 + 178 +do_execsql_test 10.1 { 179 + SELECT rowid FROM t1('b1'); 180 +} {2} 181 + 182 +do_execsql_test 10.2 { 183 + SELECT rowid FROM t1('b1') ORDER BY rowid DESC; 184 +} {2} 185 + 186 +do_execsql_test 10.3 { 187 + INSERT INTO t1(t1) VALUES('integrity-check'); 188 +} {} 189 + 190 +#-------------------------------------------------------------------------- 191 +# 192 +reset_db 193 +do_execsql_test 11.1 { 194 + CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none); 195 + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 196 + WITH d(x,y) AS ( 197 + SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d 198 + ) 199 + INSERT INTO t1 SELECT * FROM d LIMIT 23; 200 +} 201 + 202 +#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" } 203 +do_execsql_test 11.2 { 204 + SELECT rowid FROM t1; 205 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} 206 + 207 +do_execsql_test 11.3 { 208 + SELECT rowid FROM t1('xyz'); 209 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23} 210 + 211 +do_execsql_test 11.4 { 212 + INSERT INTO t1(t1) VALUES('integrity-check'); 213 +} 214 + 215 +#------------------------------------------------------------------------- 216 +# 217 +reset_db 218 +do_execsql_test 12.0 { 219 + CREATE VIRTUAL TABLE yy USING fts5(x, detail=none); 220 + INSERT INTO yy VALUES('in if'); 221 + INSERT INTO yy VALUES('if'); 222 +} {} 223 + 224 +do_execsql_test 12.1 { 225 + SELECT rowid FROM yy('i*'); 226 +} {1 2} 227 + 228 +#------------------------------------------------------------------------- 229 +# 230 +reset_db 231 +do_execsql_test 13.0 { 232 + CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none); 233 +} {} 234 +foreach {rowid a} { 235 + 0 {f} 236 + 1 {u} 237 + 2 {k} 238 + 3 {a} 239 + 4 {a} 240 + 5 {u} 241 + 6 {u} 242 + 7 {u} 243 + 8 {f} 244 + 9 {f} 245 + 10 {a} 246 + 11 {p} 247 + 12 {f} 248 + 13 {u} 249 + 14 {a} 250 + 15 {a} 251 +} { 252 + do_execsql_test 13.1.$rowid { 253 + INSERT INTO t1(rowid, a) VALUES($rowid, $a); 254 + } 255 +} 256 + 257 +#------------------------------------------------------------------------- 258 +# 259 +reset_db 260 +fts5_aux_test_functions db 261 +do_execsql_test 14.0 { 262 + CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none); 263 + INSERT INTO t1 VALUES('a b c d'); 264 +} {} 265 + 266 +do_execsql_test 14.1 { 267 + SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank; 268 +} {0.0.1} 269 + 270 +} 271 + 272 +#------------------------------------------------------------------------- 273 +# 274 +reset_db 275 +do_execsql_test 15.1 { 276 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none); 277 + BEGIN; 278 + INSERT INTO t1(rowid, x) VALUES(1, 'sqlite'); 279 + INSERT INTO t1(rowid, x) VALUES(2, 'sqlite'); 280 + COMMIT; 281 +} {} 282 + 283 +do_test 15.1 { 284 + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } 285 +} {} 286 + 287 +do_test 15.2 { 288 + execsql { DELETE FROM t1 } 289 +} {} 290 + 291 +do_execsql_test 15.3.1 { 292 + SELECT rowid FROM t1('sqlite'); 293 +} {} 294 + 295 +do_execsql_test 15.3.2 { 296 + SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC; 297 +} {} 298 + 299 +do_test 15.4 { 300 + execsql { INSERT INTO t1(t1) VALUES('integrity-check') } 301 +} {} 302 + 303 +finish_test 304 +
Added ext/fts5/test/fts5synonym2.test.
1 +# 2014 Dec 20 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# Tests focusing on custom tokenizers that support synonyms. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5synonym2 17 + 18 +# If SQLITE_ENABLE_FTS5 is defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +#------------------------------------------------------------------------- 25 +# Code for a simple Tcl tokenizer that supports synonyms at query time. 26 +# 27 +foreach SYNDICT { 28 + {zero 0} 29 + {one 1 i} 30 + {two 2 ii} 31 + {three 3 iii} 32 + {four 4 iv} 33 + {five 5 v} 34 + {six 6 vi} 35 + {seven 7 vii} 36 + {eight 8 viii} 37 + {nine 9 ix} 38 +} { 39 + foreach s $SYNDICT { 40 + set o [list] 41 + foreach x $SYNDICT {if {$x!=$s} {lappend o $x}} 42 + set ::syn($s) $o 43 + } 44 +} 45 + 46 +proc tcl_tokenize {tflags text} { 47 + foreach {w iStart iEnd} [fts5_tokenize_split $text] { 48 + sqlite3_fts5_token $w $iStart $iEnd 49 + if {$tflags == "query"} { 50 + foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd } 51 + } 52 + } 53 +} 54 + 55 +proc tcl_create {args} { 56 + return "tcl_tokenize" 57 +} 58 + 59 +# 60 +# End of tokenizer code. 61 +#------------------------------------------------------------------------- 62 + 63 +foreach_detail_mode $testprefix { 64 + 65 +sqlite3_fts5_create_tokenizer db tcl tcl_create 66 +fts5_aux_test_functions db 67 + 68 +do_execsql_test 1.0 { 69 + CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%); 70 + INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i'); 71 + INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi'); 72 + INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3'); 73 + INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7'); 74 + INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii'); 75 + INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4'); 76 + INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4'); 77 + INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii'); 78 + INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five'); 79 + INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine'); 80 + INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven'); 81 + INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three'); 82 + INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9'); 83 + INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight'); 84 + INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven'); 85 + INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four'); 86 + INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2'); 87 + INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six'); 88 + INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi'); 89 + INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi'); 90 + INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii'); 91 + INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii'); 92 + INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4'); 93 + INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii'); 94 + INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4'); 95 + INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii'); 96 + INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four'); 97 + INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five'); 98 + INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one'); 99 + INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8'); 100 + INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i'); 101 + INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9'); 102 + INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five'); 103 + INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6'); 104 + INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4'); 105 + INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv'); 106 + INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine'); 107 + INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five'); 108 + INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi'); 109 + INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii'); 110 + INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two'); 111 + INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one'); 112 + INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii'); 113 + INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4'); 114 + INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii'); 115 + INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii'); 116 + INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4'); 117 + INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii'); 118 + INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one'); 119 +} 120 + 121 +foreach {tn expr} { 122 + 1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four" 123 + 1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii" 124 + 1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2" 125 + 126 + 2.1 "one OR two OR three OR four" 127 + 2.2 "(one AND two) OR (three AND four)" 128 + 2.3 "(one AND two) OR (three AND four) NOT five" 129 + 2.4 "(one AND two) NOT 6" 130 + 131 + 3.1 "b:one AND a:two" 132 + 3.2 "b:one OR a:two" 133 + 3.3 "a:one OR b:1 OR {a b} : i" 134 + 135 + 4.1 "NEAR(one two, 2)" 136 + 4.2 "NEAR(one two three, 2)" 137 + 4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)" 138 +} { 139 + if {[fts5_expr_ok $expr ss]==0} { 140 + do_test 1.$tn.OMITTED { list } [list] 141 + continue 142 + } 143 + 144 + set res [fts5_query_data $expr ss ASC ::syn] 145 + breakpoint 146 + do_execsql_test 1.$tn.[llength $res].asc { 147 + SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr) 148 + } $res 149 +} 150 + 151 +} 152 + 153 +finish_test 154 +
Changes to ext/fts5/test/fts5vocab.test.
17 17 18 18 # If SQLITE_ENABLE_FTS5 is defined, omit this file. 19 19 ifcapable !fts5 { 20 20 finish_test 21 21 return 22 22 } 23 23 24 +foreach_detail_mode $testprefix { 25 + 26 +proc null_list_entries {iFirst nInterval L} { 27 + for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} { 28 + lset L $i {} 29 + } 30 + return $L 31 +} 32 + 33 +proc star_from_row {L} { 34 + if {[detail_is_full]==0} { 35 + set L [null_list_entries 2 3 $L] 36 + } 37 + return $L 38 +} 39 + 40 +proc star_from_col {L} { 41 + if {[detail_is_col]} { 42 + set L [null_list_entries 3 4 $L] 43 + } 44 + if {[detail_is_none]} { 45 + set L [null_list_entries 1 4 $L] 46 + set L [null_list_entries 3 4 $L] 47 + } 48 + return $L 49 +} 50 + 51 +proc row_to_col {L} { 52 + if {[detail_is_none]==0} { error "this is for detail=none mode" } 53 + set ret [list] 54 + foreach {a b c} $L { 55 + lappend ret $a {} $b {} 56 + } 57 + set ret 58 +} 24 59 25 60 do_execsql_test 1.1.1 { 26 - CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); 61 + CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%); 27 62 CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row'); 28 63 PRAGMA table_info = v1; 29 64 } { 30 65 0 term {} 0 {} 0 31 66 1 doc {} 0 {} 0 32 67 2 cnt {} 0 {} 0 33 68 } ................................................................................ 48 83 do_execsql_test 1.3 { 49 84 INSERT INTO t1 VALUES('x y z'); 50 85 INSERT INTO t1 VALUES('x x x'); 51 86 } 52 87 53 88 do_execsql_test 1.4.1 { 54 89 SELECT * FROM v1; 55 -} {x 2 4 y 1 1 z 1 1} 90 +} [star_from_row {x 2 4 y 1 1 z 1 1}] 56 91 57 92 do_execsql_test 1.4.2 { 58 93 SELECT * FROM v2; 59 -} {x one 2 4 y one 1 1 z one 1 1} 94 +} [star_from_col {x one 2 4 y one 1 1 z one 1 1}] 60 95 61 96 do_execsql_test 1.5.1 { 62 97 BEGIN; 63 98 INSERT INTO t1 VALUES('a b c'); 64 99 SELECT * FROM v1 WHERE term<'d'; 65 -} {a 1 1 b 1 1 c 1 1} 100 +} [star_from_row {a 1 1 b 1 1 c 1 1}] 66 101 67 102 do_execsql_test 1.5.2 { 68 103 SELECT * FROM v2 WHERE term<'d'; 69 104 COMMIT; 70 -} {a one 1 1 b one 1 1 c one 1 1} 105 +} [star_from_col {a one 1 1 b one 1 1 c one 1 1}] 71 106 72 107 do_execsql_test 1.6 { 73 108 DELETE FROM t1 WHERE one = 'a b c'; 74 109 SELECT * FROM v1; 75 -} {x 2 4 y 1 1 z 1 1} 110 +} [star_from_row {x 2 4 y 1 1 z 1 1}] 76 111 77 112 #------------------------------------------------------------------------- 78 113 # 79 114 do_execsql_test 2.0 { 80 - CREATE VIRTUAL TABLE tt USING fts5(a, b); 115 + CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%); 81 116 INSERT INTO tt VALUES('d g b f d f', 'f c e c d a'); 82 117 INSERT INTO tt VALUES('f a e a a b', 'e d c f d d'); 83 118 INSERT INTO tt VALUES('b c a a a b', 'f f c c b c'); 84 119 INSERT INTO tt VALUES('f d c a c e', 'd g d e g d'); 85 120 INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b'); 86 121 INSERT INTO tt VALUES('g c f b c g', 'a g f d c b'); 87 122 INSERT INTO tt VALUES('c e c f g b', 'f e d b g a'); 88 123 INSERT INTO tt VALUES('g d e f d e', 'a c d b a g'); 89 124 INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y'); 90 125 INSERT INTO tt VALUES('c c a a c f', 'd g a e b g'); 91 126 } 92 127 93 -set res_col { 128 +set res_row [star_from_row { 129 + a 10 20 b 9 14 c 9 20 d 9 19 130 + e 8 13 f 10 20 g 7 14 x 1 1 131 + y 1 1 132 +}] 133 +set res_col [star_from_col { 94 134 a a 6 11 a b 7 9 95 135 b a 6 7 b b 7 7 96 136 c a 6 12 c b 5 8 97 137 d a 4 6 d b 9 13 98 138 e a 6 7 e b 6 6 99 139 f a 9 10 f b 7 10 100 140 g a 5 7 g b 5 7 101 141 x a 1 1 y b 1 1 102 -} 103 -set res_row { 104 - a 10 20 b 9 14 c 9 20 d 9 19 105 - e 8 13 f 10 20 g 7 14 x 1 1 106 - y 1 1 142 +}] 143 +if {[detail_is_none]} { 144 + set res_col [row_to_col $res_row] 107 145 } 108 146 109 147 foreach {tn tbl resname} { 110 148 1 "fts5vocab(tt, 'col')" res_col 111 149 2 "fts5vocab(tt, 'row')" res_row 112 150 3 "fts5vocab(tt, \"row\")" res_row 113 151 4 "fts5vocab(tt, [row])" res_row ................................................................................ 149 187 #------------------------------------------------------------------------- 150 188 # Test fts5vocab tables created in the temp schema. 151 189 # 152 190 reset_db 153 191 forcedelete test.db2 154 192 do_execsql_test 5.0 { 155 193 ATTACH 'test.db2' AS aux; 156 - CREATE VIRTUAL TABLE t1 USING fts5(x); 157 - CREATE VIRTUAL TABLE temp.t1 USING fts5(x); 158 - CREATE VIRTUAL TABLE aux.t1 USING fts5(x); 194 + CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%); 195 + CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%); 196 + CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%); 159 197 160 198 INSERT INTO main.t1 VALUES('a b c'); 161 199 INSERT INTO main.t1 VALUES('d e f'); 162 200 INSERT INTO main.t1 VALUES('a e c'); 163 201 164 202 INSERT INTO temp.t1 VALUES('1 2 3'); 165 203 INSERT INTO temp.t1 VALUES('4 5 6'); ................................................................................ 174 212 do_execsql_test 5.1 { 175 213 CREATE VIRTUAL TABLE temp.vm USING fts5vocab(main, t1, row); 176 214 CREATE VIRTUAL TABLE temp.vt1 USING fts5vocab(t1, row); 177 215 CREATE VIRTUAL TABLE temp.vt2 USING fts5vocab(temp, t1, row); 178 216 CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row); 179 217 } 180 218 181 -do_execsql_test 5.2 { SELECT * FROM vm } { 219 +do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row { 182 220 a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1 183 -} 184 -do_execsql_test 5.3 { SELECT * FROM vt1 } { 221 +}] 222 +do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row { 185 223 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 186 -} 187 -do_execsql_test 5.4 { SELECT * FROM vt2 } { 224 +}] 225 +do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row { 188 226 1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1 189 -} 190 -do_execsql_test 5.5 { SELECT * FROM va } { 227 +}] 228 +do_execsql_test 5.5 { SELECT * FROM va } [star_from_row { 191 229 m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2 192 -} 230 +}] 193 231 194 232 #------------------------------------------------------------------------- 195 233 # 196 234 do_execsql_test 6.0 { 197 235 CREATE TABLE iii(iii); 198 236 CREATE TABLE jjj(x); 199 237 } ................................................................................ 214 252 } {1 {no such fts5 table: main.lll}} 215 253 216 254 #------------------------------------------------------------------------- 217 255 # Test single term queries on fts5vocab tables (i.e. those with term=? 218 256 # constraints in the WHERE clause). 219 257 # 220 258 do_execsql_test 7.0 { 221 - CREATE VIRTUAL TABLE tx USING fts5(one, two); 259 + CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%); 222 260 INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee'); 223 261 INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f'); 224 262 INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd'); 225 263 INSERT INTO tx VALUES('e bb h jjj ii gg', 'e aa e f c fff'); 226 264 INSERT INTO tx VALUES('j ff aa a h', 'h a j bbb bb'); 227 265 INSERT INTO tx VALUES('cc i ff c d f', 'dd ii fff f c cc d'); 228 266 INSERT INTO tx VALUES('jjj g i bb cc eee', 'hhh iii aaa b bbb aaa'); ................................................................................ 272 310 273 311 set r2 [db eval { 274 312 SELECT $term, 'two', sum(cont(two, $term)>0), sum(cont(two, $term)) FROM tx 275 313 }] 276 314 if {[lindex $r2 2]==0} {set r2 [list]} 277 315 278 316 set resc [concat $r1 $r2] 317 + 318 + set resc [star_from_col $resc] 319 + set resr [star_from_row $resr] 320 + if {[detail_is_none]} { set resc [row_to_col $resr] } 279 321 do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc 280 322 do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr 281 323 } 282 324 283 325 do_execsql_test 7.1 { 284 326 CREATE TABLE txr_c AS SELECT * FROM txr; 285 327 CREATE TABLE txc_c AS SELECT * FROM txc; ................................................................................ 336 378 } [db eval {SELECT * FROM txc_c WHERE term>$a AND term <$b}] 337 379 } 338 380 339 381 do_execsql_test 7.3.1 { 340 382 SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term; 341 383 } {30} 342 384 343 -do_execsql_test 7.3.2 { 344 - SELECT count(*) FROM txc, txc_c 345 - WHERE txc.term = txc_c.term AND txc.col=txc_c.col; 346 -} {57} 385 +if {![detail_is_none]} { 386 + do_execsql_test 7.3.2 { 387 + SELECT count(*) FROM txc, txc_c 388 + WHERE txc.term = txc_c.term AND txc.col=txc_c.col; 389 + } {57} 390 +} 391 + 392 +} 347 393 348 394 finish_test 349 395
Added ext/fts5/tool/fts5speed.tcl.
1 + 2 + 3 +set Q { 4 + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"} 5 + {25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"} 6 + {300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"} 7 + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"} 8 + {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"} 9 + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"} 10 + 11 + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"} 12 + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"} 13 + {1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"} 14 +} 15 + 16 +proc usage {} { 17 + global Q 18 + puts stderr "Usage: $::argv0 DATABASE QUERY" 19 + puts stderr "" 20 + for {set i 1} {$i <= [llength $Q]} {incr i} { 21 + puts stderr " $i. [lindex $Q [expr $i-1]]" 22 + } 23 + puts stderr "" 24 + exit -1 25 +} 26 + 27 + 28 +set nArg [llength $argv] 29 +if {$nArg!=2 && $nArg!=3} usage 30 +set database [lindex $argv 0] 31 +set iquery [lindex $argv 1] 32 +if {$iquery<1 || $iquery>[llength $Q]} usage 33 +set nRepeat 0 34 +if {$nArg==3} { set nRepeat [lindex $argv 2] } 35 + 36 + 37 +sqlite3 db $database 38 +catch { load_static_extension db fts5 } 39 + 40 +incr iquery -1 41 +set sql [lindex $Q $iquery 1] 42 +if {$nRepeat==0} { 43 + set nRepeat [lindex $Q $iquery 0] 44 +} 45 + 46 +puts "sql: $sql" 47 +puts "nRepeat: $nRepeat" 48 +if {[regexp matchinfo $sql]} { 49 + sqlite3_fts5_register_matchinfo db 50 + db eval $sql 51 +} else { 52 + puts "result: [db eval $sql]" 53 +} 54 + 55 +for {set i 1} {$i < $nRepeat} {incr i} { 56 + db eval $sql 57 +} 58 + 59 +
Changes to ext/rbu/rbu.c.
71 71 int nStep = 0; /* Maximum number of step() calls */ 72 72 int rc; 73 73 sqlite3_int64 nProgress = 0; 74 74 75 75 /* Process command line arguments. Following this block local variables 76 76 ** zTarget, zRbu and nStep are all set. */ 77 77 if( argc==5 ){ 78 - int nArg1 = strlen(argv[1]); 78 + size_t nArg1 = strlen(argv[1]); 79 79 if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); 80 80 nStep = atoi(argv[2]); 81 81 }else if( argc!=3 ){ 82 82 usage(argv[0]); 83 83 } 84 84 zTarget = argv[argc-2]; 85 85 zRbu = argv[argc-1]; ................................................................................ 99 99 /* Let the user know what happened. */ 100 100 switch( rc ){ 101 101 case SQLITE_OK: 102 102 sqlite3_snprintf(sizeof(zBuf), zBuf, 103 103 "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", 104 104 nProgress 105 105 ); 106 - fprintf(stdout, zBuf); 106 + fprintf(stdout, "%s", zBuf); 107 107 break; 108 108 109 109 case SQLITE_DONE: 110 110 sqlite3_snprintf(sizeof(zBuf), zBuf, 111 111 "SQLITE_DONE: rbu update completed (%lld operations)\n", 112 112 nProgress 113 113 ); 114 - fprintf(stdout, zBuf); 114 + fprintf(stdout, "%s", zBuf); 115 115 break; 116 116 117 117 default: 118 118 fprintf(stderr, "error=%d: %s\n", rc, zErrmsg); 119 119 break; 120 120 } 121 121 122 122 sqlite3_free(zErrmsg); 123 123 return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1; 124 124 } 125 -
Changes to ext/rbu/sqlite3rbu.c.
931 931 ** immediately without attempting the allocation or modifying the stored 932 932 ** error code. 933 933 */ 934 934 static void *rbuMalloc(sqlite3rbu *p, int nByte){ 935 935 void *pRet = 0; 936 936 if( p->rc==SQLITE_OK ){ 937 937 assert( nByte>0 ); 938 - pRet = sqlite3_malloc(nByte); 938 + pRet = sqlite3_malloc64(nByte); 939 939 if( pRet==0 ){ 940 940 p->rc = SQLITE_NOMEM; 941 941 }else{ 942 942 memset(pRet, 0, nByte); 943 943 } 944 944 } 945 945 return pRet; ................................................................................ 977 977 ** if the allocation succeeds, (*pRc) is left unchanged. 978 978 */ 979 979 static char *rbuStrndup(const char *zStr, int *pRc){ 980 980 char *zRet = 0; 981 981 982 982 assert( *pRc==SQLITE_OK ); 983 983 if( zStr ){ 984 - int nCopy = strlen(zStr) + 1; 985 - zRet = (char*)sqlite3_malloc(nCopy); 984 + size_t nCopy = strlen(zStr) + 1; 985 + zRet = (char*)sqlite3_malloc64(nCopy); 986 986 if( zRet ){ 987 987 memcpy(zRet, zStr, nCopy); 988 988 }else{ 989 989 *pRc = SQLITE_NOMEM; 990 990 } 991 991 } 992 992 ................................................................................ 2326 2326 return SQLITE_INTERNAL; 2327 2327 } 2328 2328 2329 2329 pRbu->pgsz = iAmt; 2330 2330 if( pRbu->nFrame==pRbu->nFrameAlloc ){ 2331 2331 int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; 2332 2332 RbuFrame *aNew; 2333 - aNew = (RbuFrame*)sqlite3_realloc(pRbu->aFrame, nNew * sizeof(RbuFrame)); 2333 + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); 2334 2334 if( aNew==0 ) return SQLITE_NOMEM; 2335 2335 pRbu->aFrame = aNew; 2336 2336 pRbu->nFrameAlloc = nNew; 2337 2337 } 2338 2338 2339 2339 iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; 2340 2340 if( pRbu->iMaxFrame<iFrame ) pRbu->iMaxFrame = iFrame; ................................................................................ 2391 2391 int nChar; 2392 2392 LPWSTR zWideFilename; 2393 2393 2394 2394 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); 2395 2395 if( nChar==0 ){ 2396 2396 return 0; 2397 2397 } 2398 - zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); 2398 + zWideFilename = sqlite3_malloc64( nChar*sizeof(zWideFilename[0]) ); 2399 2399 if( zWideFilename==0 ){ 2400 2400 return 0; 2401 2401 } 2402 2402 memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0])); 2403 2403 nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, 2404 2404 nChar); 2405 2405 if( nChar==0 ){ ................................................................................ 3025 3025 */ 3026 3026 sqlite3rbu *sqlite3rbu_open( 3027 3027 const char *zTarget, 3028 3028 const char *zRbu, 3029 3029 const char *zState 3030 3030 ){ 3031 3031 sqlite3rbu *p; 3032 - int nTarget = strlen(zTarget); 3033 - int nRbu = strlen(zRbu); 3034 - int nState = zState ? strlen(zState) : 0; 3032 + size_t nTarget = strlen(zTarget); 3033 + size_t nRbu = strlen(zRbu); 3034 + size_t nState = zState ? strlen(zState) : 0; 3035 + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; 3035 3036 3036 - p = (sqlite3rbu*)sqlite3_malloc(sizeof(sqlite3rbu)+nTarget+1+nRbu+1+nState+1); 3037 + p = (sqlite3rbu*)sqlite3_malloc64(nByte); 3037 3038 if( p ){ 3038 3039 RbuState *pState = 0; 3039 3040 3040 3041 /* Create the custom VFS. */ 3041 3042 memset(p, 0, sizeof(sqlite3rbu)); 3042 3043 rbuCreateVfs(p); 3043 3044 ................................................................................ 3166 3167 ** If the error code currently stored in the RBU handle is SQLITE_CONSTRAINT, 3167 3168 ** then edit any error message string so as to remove all occurrences of 3168 3169 ** the pattern "rbu_imp_[0-9]*". 3169 3170 */ 3170 3171 static void rbuEditErrmsg(sqlite3rbu *p){ 3171 3172 if( p->rc==SQLITE_CONSTRAINT && p->zErrmsg ){ 3172 3173 int i; 3173 - int nErrmsg = strlen(p->zErrmsg); 3174 + size_t nErrmsg = strlen(p->zErrmsg); 3174 3175 for(i=0; i<(nErrmsg-8); i++){ 3175 3176 if( memcmp(&p->zErrmsg[i], "rbu_imp_", 8)==0 ){ 3176 3177 int nDel = 8; 3177 3178 while( p->zErrmsg[i+nDel]>='0' && p->zErrmsg[i+nDel]<='9' ) nDel++; 3178 3179 memmove(&p->zErrmsg[i], &p->zErrmsg[i+nDel], nErrmsg + 1 - i - nDel); 3179 3180 nErrmsg -= nDel; 3180 3181 } ................................................................................ 3630 3631 /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this 3631 3632 ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space 3632 3633 ** instead of a file on disk. */ 3633 3634 assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); 3634 3635 if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ 3635 3636 if( iRegion<=p->nShm ){ 3636 3637 int nByte = (iRegion+1) * sizeof(char*); 3637 - char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); 3638 + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); 3638 3639 if( apNew==0 ){ 3639 3640 rc = SQLITE_NOMEM; 3640 3641 }else{ 3641 3642 memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); 3642 3643 p->apShm = apNew; 3643 3644 p->nShm = iRegion+1; 3644 3645 } 3645 3646 } 3646 3647 3647 3648 if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ 3648 - char *pNew = (char*)sqlite3_malloc(szRegion); 3649 + char *pNew = (char*)sqlite3_malloc64(szRegion); 3649 3650 if( pNew==0 ){ 3650 3651 rc = SQLITE_NOMEM; 3651 3652 }else{ 3652 3653 memset(pNew, 0, szRegion); 3653 3654 p->apShm[iRegion] = pNew; 3654 3655 } 3655 3656 } ................................................................................ 3751 3752 if( zName ){ 3752 3753 if( flags & SQLITE_OPEN_MAIN_DB ){ 3753 3754 /* A main database has just been opened. The following block sets 3754 3755 ** (pFd->zWal) to point to a buffer owned by SQLite that contains 3755 3756 ** the name of the *-wal file this db connection will use. SQLite 3756 3757 ** happens to pass a pointer to this buffer when using xAccess() 3757 3758 ** or xOpen() to operate on the *-wal file. */ 3758 - int n = strlen(zName); 3759 + int n = (int)strlen(zName); 3759 3760 const char *z = &zName[n]; 3760 3761 if( flags & SQLITE_OPEN_URI ){ 3761 3762 int odd = 0; 3762 3763 while( 1 ){ 3763 3764 if( z[0]==0 ){ 3764 3765 odd = 1 - odd; 3765 3766 if( odd && z[1]==0 ) break; ................................................................................ 3777 3778 rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); 3778 3779 if( pDb ){ 3779 3780 if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ 3780 3781 /* This call is to open a *-wal file. Intead, open the *-oal. This 3781 3782 ** code ensures that the string passed to xOpen() is terminated by a 3782 3783 ** pair of '\0' bytes in case the VFS attempts to extract a URI 3783 3784 ** parameter from it. */ 3784 - int nCopy = strlen(zName); 3785 - char *zCopy = sqlite3_malloc(nCopy+2); 3785 + size_t nCopy = strlen(zName); 3786 + char *zCopy = sqlite3_malloc64(nCopy+2); 3786 3787 if( zCopy ){ 3787 3788 memcpy(zCopy, zName, nCopy); 3788 3789 zCopy[nCopy-3] = 'o'; 3789 3790 zCopy[nCopy] = '\0'; 3790 3791 zCopy[nCopy+1] = '\0'; 3791 3792 zOpen = (const char*)(pFd->zDel = zCopy); 3792 3793 }else{ ................................................................................ 4007 4008 rbuVfsCurrentTime, /* xCurrentTime */ 4008 4009 rbuVfsGetLastError, /* xGetLastError */ 4009 4010 0, /* xCurrentTimeInt64 (version 2) */ 4010 4011 0, 0, 0 /* Unimplemented version 3 methods */ 4011 4012 }; 4012 4013 4013 4014 rbu_vfs *pNew = 0; /* Newly allocated VFS */ 4014 - int nName; 4015 4015 int rc = SQLITE_OK; 4016 + size_t nName; 4017 + size_t nByte; 4016 4018 4017 - int nByte; 4018 4019 nName = strlen(zName); 4019 4020 nByte = sizeof(rbu_vfs) + nName + 1; 4020 - pNew = (rbu_vfs*)sqlite3_malloc(nByte); 4021 + pNew = (rbu_vfs*)sqlite3_malloc64(nByte); 4021 4022 if( pNew==0 ){ 4022 4023 rc = SQLITE_NOMEM; 4023 4024 }else{ 4024 4025 sqlite3_vfs *pParent; /* Parent VFS */ 4025 4026 memset(pNew, 0, nByte); 4026 4027 pParent = sqlite3_vfs_find(zParent); 4027 4028 if( pParent==0 ){
Changes to src/btree.c.
1081 1081 static void btreeParseCellPtrNoPayload( 1082 1082 MemPage *pPage, /* Page containing the cell */ 1083 1083 u8 *pCell, /* Pointer to the cell text. */ 1084 1084 CellInfo *pInfo /* Fill in this structure */ 1085 1085 ){ 1086 1086 assert( sqlite3_mutex_held(pPage->pBt->mutex) ); 1087 1087 assert( pPage->leaf==0 ); 1088 - assert( pPage->noPayload ); 1089 1088 assert( pPage->childPtrSize==4 ); 1090 1089 #ifndef SQLITE_DEBUG 1091 1090 UNUSED_PARAMETER(pPage); 1092 1091 #endif 1093 1092 pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); 1094 1093 pInfo->nPayload = 0; 1095 1094 pInfo->nLocal = 0; ................................................................................ 1103 1102 ){ 1104 1103 u8 *pIter; /* For scanning through pCell */ 1105 1104 u32 nPayload; /* Number of bytes of cell payload */ 1106 1105 u64 iKey; /* Extracted Key value */ 1107 1106 1108 1107 assert( sqlite3_mutex_held(pPage->pBt->mutex) ); 1109 1108 assert( pPage->leaf==0 || pPage->leaf==1 ); 1110 - assert( pPage->intKeyLeaf || pPage->noPayload ); 1111 - assert( pPage->noPayload==0 ); 1112 1109 assert( pPage->intKeyLeaf ); 1113 1110 assert( pPage->childPtrSize==0 ); 1114 1111 pIter = pCell; 1115 1112 1116 1113 /* The next block of code is equivalent to: 1117 1114 ** 1118 1115 ** pIter += getVarint32(pIter, nPayload); ................................................................................ 1173 1170 ){ 1174 1171 u8 *pIter; /* For scanning through pCell */ 1175 1172 u32 nPayload; /* Number of bytes of cell payload */ 1176 1173 1177 1174 assert( sqlite3_mutex_held(pPage->pBt->mutex) ); 1178 1175 assert( pPage->leaf==0 || pPage->leaf==1 ); 1179 1176 assert( pPage->intKeyLeaf==0 ); 1180 - assert( pPage->noPayload==0 ); 1181 1177 pIter = pCell + pPage->childPtrSize; 1182 1178 nPayload = *pIter; 1183 1179 if( nPayload>=0x80 ){ 1184 1180 u8 *pEnd = &pIter[8]; 1185 1181 nPayload &= 0x7f; 1186 1182 do{ 1187 1183 nPayload = (nPayload<<7) | (*++pIter & 0x7f); ................................................................................ 1234 1230 ** the (CellInfo.nSize) value found by doing a full parse of the 1235 1231 ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of 1236 1232 ** this function verifies that this invariant is not violated. */ 1237 1233 CellInfo debuginfo; 1238 1234 pPage->xParseCell(pPage, pCell, &debuginfo); 1239 1235 #endif 1240 1236 1241 - assert( pPage->noPayload==0 ); 1242 1237 nSize = *pIter; 1243 1238 if( nSize>=0x80 ){ 1244 1239 pEnd = &pIter[8]; 1245 1240 nSize &= 0x7f; 1246 1241 do{ 1247 1242 nSize = (nSize<<7) | (*++pIter & 0x7f); 1248 1243 }while( *(pIter)>=0x80 && pIter<pEnd ); ................................................................................ 1692 1687 assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); 1693 1688 /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf 1694 1689 ** table b-tree page. */ 1695 1690 assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); 1696 1691 pPage->intKey = 1; 1697 1692 if( pPage->leaf ){ 1698 1693 pPage->intKeyLeaf = 1; 1699 - pPage->noPayload = 0; 1700 1694 pPage->xParseCell = btreeParseCellPtr; 1701 1695 }else{ 1702 1696 pPage->intKeyLeaf = 0; 1703 - pPage->noPayload = 1; 1704 1697 pPage->xCellSize = cellSizePtrNoPayload; 1705 1698 pPage->xParseCell = btreeParseCellPtrNoPayload; 1706 1699 } 1707 1700 pPage->maxLocal = pBt->maxLeaf; 1708 1701 pPage->minLocal = pBt->minLeaf; 1709 1702 }else if( flagByte==PTF_ZERODATA ){ 1710 1703 /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior ................................................................................ 1711 1704 ** index b-tree page. */ 1712 1705 assert( (PTF_ZERODATA)==2 ); 1713 1706 /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf 1714 1707 ** index b-tree page. */ 1715 1708 assert( (PTF_ZERODATA|PTF_LEAF)==10 ); 1716 1709 pPage->intKey = 0; 1717 1710 pPage->intKeyLeaf = 0; 1718 - pPage->noPayload = 0; 1719 1711 pPage->xParseCell = btreeParseCellPtrIndex; 1720 1712 pPage->maxLocal = pBt->maxLocal; 1721 1713 pPage->minLocal = pBt->minLocal; 1722 1714 }else{ 1723 1715 /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is 1724 1716 ** an error. */ 1725 1717 return SQLITE_CORRUPT_BKPT;
Changes to src/btreeInt.h.
272 272 ** stored in MemPage.pBt->mutex. 273 273 */ 274 274 struct MemPage { 275 275 u8 isInit; /* True if previously initialized. MUST BE FIRST! */ 276 276 u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ 277 277 u8 intKey; /* True if table b-trees. False for index b-trees */ 278 278 u8 intKeyLeaf; /* True if the leaf of an intKey table */ 279 - u8 noPayload; /* True if internal intKey page (thus w/o data) */ 280 279 u8 leaf; /* True if a leaf page */ 281 280 u8 hdrOffset; /* 100 for page 1. 0 otherwise */ 282 281 u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ 283 282 u8 max1bytePayload; /* min(maxLocal,127) */ 284 283 u8 bBusy; /* Prevent endless loops on corrupt database files */ 285 284 u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ 286 285 u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */
Changes to src/main.c.
3539 3539 assert( fd!=0 ); 3540 3540 if( op==SQLITE_FCNTL_FILE_POINTER ){ 3541 3541 *(sqlite3_file**)pArg = fd; 3542 3542 rc = SQLITE_OK; 3543 3543 }else if( op==SQLITE_FCNTL_VFS_POINTER ){ 3544 3544 *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); 3545 3545 rc = SQLITE_OK; 3546 + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ 3547 + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); 3548 + rc = SQLITE_OK; 3546 3549 }else if( fd->pMethods ){ 3547 3550 rc = sqlite3OsFileControl(fd, op, pArg); 3548 3551 #ifndef SQLITE_OMIT_WAL 3549 3552 if( (rc==SQLITE_OK)&&(op==SQLITE_FCNTL_LAST_ERRNO)&&(*(int *)pArg==0) ){ 3550 3553 sqlite3_file *pWalFd = sqlite3PagerWalFile(pPager); 3551 3554 if( pWalFd&&(pWalFd->pMethods) ){ 3552 3555 rc = sqlite3OsFileControl(pWalFd, op, pArg);
Changes to src/os_unix.c.
714 714 715 715 { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, 716 716 #define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) 717 717 718 718 { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, 719 719 #define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) 720 720 721 +#if defined(HAVE_FCHOWN) 721 722 { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, 723 +#else 724 + { "fchown", (sqlite3_syscall_ptr)0, 0 }, 725 +#endif 722 726 #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) 723 727 724 728 { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, 725 729 #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) 726 730 727 731 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 728 732 { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, ................................................................................ 748 752 #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 749 753 { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, 750 754 #else 751 755 { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, 752 756 #endif 753 757 #define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) 754 758 759 +#if defined(HAVE_READLINK) 755 760 { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, 761 +#else 762 + { "readlink", (sqlite3_syscall_ptr)0, 0 }, 763 +#endif 756 764 #define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) 757 765 758 766 759 767 }; /* End of the overrideable system calls */ 760 768 761 769 762 770 /* 763 771 ** On some systems, calls to fchown() will trigger a message in a security 764 772 ** log if they come from non-root processes. So avoid calling fchown() if 765 773 ** we are not running as root. 766 774 */ 767 775 static int robustFchown(int fd, uid_t uid, gid_t gid){ 768 -#if OS_VXWORKS 769 - return 0; 770 -#else 776 +#if defined(HAVE_FCHOWN) 771 777 return osGeteuid() ? 0 : osFchown(fd,uid,gid); 778 +#else 779 + return 0; 772 780 #endif 773 781 } 774 782 775 783 /* 776 784 ** This is the xSetSystemCall() method of sqlite3_vfs for all of the 777 785 ** "unix" VFSes. Return SQLITE_OK opon successfully updating the 778 786 ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ................................................................................ 7150 7158 ** current working directory has been unlinked. 7151 7159 */ 7152 7160 SimulateIOError( return SQLITE_ERROR ); 7153 7161 7154 7162 assert( pVfs->mxPathname==MAX_PATHNAME ); 7155 7163 UNUSED_PARAMETER(pVfs); 7156 7164 7165 +#if defined(HAVE_READLINK) 7157 7166 /* Attempt to resolve the path as if it were a symbolic link. If it is 7158 7167 ** a symbolic link, the resolved path is stored in buffer zOut[]. Or, if 7159 7168 ** the identified file is not a symbolic link or does not exist, then 7160 7169 ** zPath is copied directly into zOut. Either way, nByte is left set to 7161 7170 ** the size of the string copied into zOut[] in bytes. */ 7162 7171 nByte = osReadlink(zPath, zOut, nOut-1); 7163 7172 if( nByte<0 ){ ................................................................................ 7165 7174 return unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zPath); 7166 7175 } 7167 7176 sqlite3_snprintf(nOut, zOut, "%s", zPath); 7168 7177 nByte = sqlite3Strlen30(zOut); 7169 7178 }else{ 7170 7179 zOut[nByte] = '\0'; 7171 7180 } 7181 +#endif 7172 7182 7173 7183 /* If buffer zOut[] now contains an absolute path there is nothing more 7174 7184 ** to do. If it contains a relative path, do the following: 7175 7185 ** 7176 7186 ** * move the relative path string so that it is at the end of th 7177 7187 ** zOut[] buffer. 7178 7188 ** * Call getcwd() to read the path of the current working directory
Changes to src/pager.c.
5649 5649 ** exclusive lock on the database is not already held, obtain it now. 5650 5650 */ 5651 5651 if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ 5652 5652 rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); 5653 5653 if( rc!=SQLITE_OK ){ 5654 5654 return rc; 5655 5655 } 5656 - sqlite3WalExclusiveMode(pPager->pWal, 1); 5656 + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); 5657 5657 } 5658 5658 5659 5659 /* Grab the write lock on the log file. If successful, upgrade to 5660 5660 ** PAGER_RESERVED state. Otherwise, return an error code to the caller. 5661 5661 ** The busy-handler is not invoked if another connection already 5662 5662 ** holds the write-lock. If possible, the upper layer will call it. 5663 5663 */ ................................................................................ 6714 6714 ** with the pager. This might return NULL if the file has 6715 6715 ** not yet been opened. 6716 6716 */ 6717 6717 sqlite3_file *sqlite3PagerFile(Pager *pPager){ 6718 6718 return pPager->fd; 6719 6719 } 6720 6720 6721 +#if !defined(SQLITE_OMIT_WAL) 6721 6722 /* 6722 6723 ** Return the file handle for the WAL journal file associated 6723 6724 ** with the pager. This might return NULL if the file has 6724 6725 ** not yet been opened. 6725 6726 */ 6726 6727 sqlite3_file *sqlite3PagerWalFile(Pager *pPager){ 6727 6728 return ((pPager->pWal) ? sqlite3WalFile(pPager->pWal) : (NULL)); 6729 +} 6730 +#endif 6731 + 6732 + 6733 +/* 6734 +** Return the file handle for the journal file (if it exists). 6735 +** This will be either the rollback journal or the WAL file. 6736 +*/ 6737 +sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ 6738 +#if SQLITE_OMIT_WAL 6739 + return pPager->jfd; 6740 +#else 6741 + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; 6742 +#endif 6728 6743 } 6729 6744 6730 6745 /* 6731 6746 ** Return the full pathname of the journal file. 6732 6747 */ 6733 6748 const char *sqlite3PagerJournalname(Pager *pPager){ 6734 6749 return pPager->zJournal;
Changes to src/pager.h.
185 185 int sqlite3PagerRefcount(Pager*); 186 186 #endif 187 187 int sqlite3PagerMemUsed(Pager*); 188 188 const char *sqlite3PagerFilename(Pager*, int); 189 189 sqlite3_vfs *sqlite3PagerVfs(Pager*); 190 190 sqlite3_file *sqlite3PagerFile(Pager*); 191 191 sqlite3_file *sqlite3PagerWalFile(Pager *pPager); 192 +sqlite3_file *sqlite3PagerJrnlFile(Pager*); 192 193 const char *sqlite3PagerJournalname(Pager*); 193 194 int sqlite3PagerNosync(Pager*); 194 195 void *sqlite3PagerTempSpace(Pager*); 195 196 int sqlite3PagerIsMemdb(Pager*); 196 197 void sqlite3PagerCacheStat(Pager *, int, int, int *); 197 198 void sqlite3PagerClearCache(Pager *); 198 199 int sqlite3SectorSize(sqlite3_file *);
Changes to src/select.c.
50 50 struct SortCtx { 51 51 ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ 52 52 int nOBSat; /* Number of ORDER BY terms satisfied by indices */ 53 53 int iECursor; /* Cursor number for the sorter */ 54 54 int regReturn; /* Register holding block-output return address */ 55 55 int labelBkOut; /* Start label for the block-output subroutine */ 56 56 int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ 57 + int labelDone; /* Jump here when done, ex: LIMIT reached */ 57 58 u8 sortFlags; /* Zero or more SORTFLAG_* bits */ 58 59 }; 59 60 #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ 60 61 61 62 /* 62 63 ** Delete all the content of a Select structure. Deallocate the structure 63 64 ** itself only if bFree is true. ................................................................................ 120 121 pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); 121 122 } 122 123 pNew->pEList = pEList; 123 124 pNew->op = TK_SELECT; 124 125 pNew->selFlags = selFlags; 125 126 pNew->iLimit = 0; 126 127 pNew->iOffset = 0; 128 +#if SELECTTRACE_ENABLED 129 + pNew->zSelName[0] = 0; 130 +#endif 127 131 pNew->addrOpenEphm[0] = -1; 128 132 pNew->addrOpenEphm[1] = -1; 129 133 pNew->nSelectRow = 0; 130 134 if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); 131 135 pNew->pSrc = pSrc; 132 136 pNew->pWhere = pWhere; 133 137 pNew->pGroupBy = pGroupBy; ................................................................................ 509 513 int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); 510 514 int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ 511 515 int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ 512 516 int regBase; /* Regs for sorter record */ 513 517 int regRecord = ++pParse->nMem; /* Assembled sorter record */ 514 518 int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ 515 519 int op; /* Opcode to add sorter record to sorter */ 520 + int iLimit; /* LIMIT counter */ 516 521 517 522 assert( bSeq==0 || bSeq==1 ); 518 523 assert( nData==1 || regData==regOrigData ); 519 524 if( nPrefixReg ){ 520 525 assert( nPrefixReg==nExpr+bSeq ); 521 526 regBase = regData - nExpr - bSeq; 522 527 }else{ 523 528 regBase = pParse->nMem + 1; 524 529 pParse->nMem += nBase; 525 530 } 531 + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); 532 + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; 533 + pSort->labelDone = sqlite3VdbeMakeLabel(v); 526 534 sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, 527 535 SQLITE_ECEL_DUP|SQLITE_ECEL_REF); 528 536 if( bSeq ){ 529 537 sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); 530 538 } 531 539 if( nPrefixReg==0 ){ 532 540 sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); 533 541 } 534 - 535 542 sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); 536 543 if( nOBSat>0 ){ 537 544 int regPrevKey; /* The first nOBSat columns of the previous row */ 538 545 int addrFirst; /* Address of the OP_IfNot opcode */ 539 546 int addrJmp; /* Address of the OP_Jump opcode */ 540 547 VdbeOp *pOp; /* Opcode that opens the sorter */ 541 548 int nKey; /* Number of sorting key columns, including OP_Sequence */ ................................................................................ 562 569 pKI->nXField-1); 563 570 addrJmp = sqlite3VdbeCurrentAddr(v); 564 571 sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); 565 572 pSort->labelBkOut = sqlite3VdbeMakeLabel(v); 566 573 pSort->regReturn = ++pParse->nMem; 567 574 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); 568 575 sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); 576 + if( iLimit ){ 577 + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); 578 + VdbeCoverage(v); 579 + } 569 580 sqlite3VdbeJumpHere(v, addrFirst); 570 581 sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); 571 582 sqlite3VdbeJumpHere(v, addrJmp); 572 583 } 573 584 if( pSort->sortFlags & SORTFLAG_UseSorter ){ 574 585 op = OP_SorterInsert; 575 586 }else{ 576 587 op = OP_IdxInsert; 577 588 } 578 589 sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); 579 - if( pSelect->iLimit ){ 590 + if( iLimit ){ 580 591 int addr; 581 - int iLimit; 582 - if( pSelect->iOffset ){ 583 - iLimit = pSelect->iOffset+1; 584 - }else{ 585 - iLimit = pSelect->iLimit; 586 - } 587 592 addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); 588 593 sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); 589 594 sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); 590 595 sqlite3VdbeJumpHere(v, addr); 591 596 } 592 597 } 593 598 ................................................................................ 1183 1188 Parse *pParse, /* Parsing context */ 1184 1189 Select *p, /* The SELECT statement */ 1185 1190 SortCtx *pSort, /* Information on the ORDER BY clause */ 1186 1191 int nColumn, /* Number of columns of data */ 1187 1192 SelectDest *pDest /* Write the sorted results here */ 1188 1193 ){ 1189 1194 Vdbe *v = pParse->pVdbe; /* The prepared statement */ 1190 - int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ 1195 + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ 1191 1196 int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ 1192 1197 int addr; 1193 1198 int addrOnce = 0; 1194 1199 int iTab; 1195 1200 ExprList *pOrderBy = pSort->pOrderBy; 1196 1201 int eDest = pDest->eDest; 1197 1202 int iParm = pDest->iSDParm; ................................................................................ 1202 1207 int nSortData; /* Trailing values to read from sorter */ 1203 1208 int i; 1204 1209 int bSeq; /* True if sorter record includes seq. no. */ 1205 1210 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS 1206 1211 struct ExprList_item *aOutEx = p->pEList->a; 1207 1212 #endif 1208 1213 1214 + assert( addrBreak<0 ); 1209 1215 if( pSort->labelBkOut ){ 1210 1216 sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); 1211 1217 sqlite3VdbeGoto(v, addrBreak); 1212 1218 sqlite3VdbeResolveLabel(v, pSort->labelBkOut); 1213 1219 } 1214 1220 iTab = pSort->iECursor; 1215 1221 if( eDest==SRT_Output || eDest==SRT_Coroutine ){
Changes to src/shell.c.
1917 1917 ".tables ?TABLE? List names of tables\n" 1918 1918 " If TABLE specified, only list tables matching\n" 1919 1919 " LIKE pattern TABLE.\n" 1920 1920 ".timeout MS Try opening locked tables for MS milliseconds\n" 1921 1921 ".timer on|off Turn SQL timer on or off\n" 1922 1922 ".trace FILE|off Output each SQL statement as it is run\n" 1923 1923 ".vfsinfo ?AUX? Information about the top-level VFS\n" 1924 + ".vfslist List all available VFSes\n" 1924 1925 ".vfsname ?AUX? Print the name of the VFS stack\n" 1925 1926 ".width NUM1 NUM2 ... Set column widths for \"column\" mode\n" 1926 1927 " Negative values right-justify\n" 1927 1928 ; 1928 1929 1929 1930 /* Forward reference */ 1930 1931 static int process_input(ShellState *p, FILE *in); ................................................................................ 4163 4164 sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFS_POINTER, &pVfs); 4164 4165 if( pVfs ){ 4165 4166 utf8_printf(p->out, "vfs.zName = \"%s\"\n", pVfs->zName); 4166 4167 raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); 4167 4168 raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); 4168 4169 raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); 4169 4170 } 4171 + } 4172 + }else 4173 + 4174 + if( c=='v' && strncmp(azArg[0], "vfslist", n)==0 ){ 4175 + sqlite3_vfs *pVfs; 4176 + sqlite3_vfs *pCurrent = 0; 4177 + if( p->db ){ 4178 + sqlite3_file_control(p->db, "main", SQLITE_FCNTL_VFS_POINTER, &pCurrent); 4179 + } 4180 + for(pVfs=sqlite3_vfs_find(0); pVfs; pVfs=pVfs->pNext){ 4181 + utf8_printf(p->out, "vfs.zName = \"%s\"%s\n", pVfs->zName, 4182 + pVfs==pCurrent ? " <--- CURRENT" : ""); 4183 + raw_printf(p->out, "vfs.iVersion = %d\n", pVfs->iVersion); 4184 + raw_printf(p->out, "vfs.szOsFile = %d\n", pVfs->szOsFile); 4185 + raw_printf(p->out, "vfs.mxPathname = %d\n", pVfs->mxPathname); 4186 + if( pVfs->pNext ){ 4187 + raw_printf(p->out, "-----------------------------------\n"); 4188 + } 4170 4189 } 4171 4190 }else 4172 4191 4173 4192 if( c=='v' && strncmp(azArg[0], "vfsname", n)==0 ){ 4174 4193 const char *zDbName = nArg==2 ? azArg[1] : "main"; 4175 4194 char *zVfsName = 0; 4176 4195 if( p->db ){
Changes to src/sqlite.h.in.
791 791 ** for the nominated database. Allocating database file space in large 792 792 ** chunks (say 1MB at a time), may reduce file-system fragmentation and 793 793 ** improve performance on some systems. 794 794 ** 795 795 ** <li>[[SQLITE_FCNTL_FILE_POINTER]] 796 796 ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer 797 797 ** to the [sqlite3_file] object associated with a particular database 798 -** connection. See the [sqlite3_file_control()] documentation for 799 -** additional information. 798 +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. 799 +** 800 +** <li>[[SQLITE_FCNTL_JOURNAL_POINTER]] 801 +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer 802 +** to the [sqlite3_file] object associated with the journal file (either 803 +** the [rollback journal] or the [write-ahead log]) for a particular database 804 +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. 800 805 ** 801 806 ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] 802 807 ** No longer in use. 803 808 ** 804 809 ** <li>[[SQLITE_FCNTL_SYNC]] 805 810 ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and 806 811 ** sent to the VFS immediately before the xSync method is invoked on a ................................................................................ 1007 1012 #define SQLITE_FCNTL_SYNC 21 1008 1013 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 1009 1014 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 1010 1015 #define SQLITE_FCNTL_WAL_BLOCK 24 1011 1016 #define SQLITE_FCNTL_ZIPVFS 25 1012 1017 #define SQLITE_FCNTL_RBU 26 1013 1018 #define SQLITE_FCNTL_VFS_POINTER 27 1019 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 1014 1020 1015 1021 /* deprecated names */ 1016 1022 #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE 1017 1023 #define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE 1018 1024 #define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO 1019 1025 1020 1026
Changes to src/vdbe.c.
2371 2371 const u8 *zData; /* Part of the record being decoded */ 2372 2372 const u8 *zHdr; /* Next unparsed byte of the header */ 2373 2373 const u8 *zEndHdr; /* Pointer to first byte after the header */ 2374 2374 u32 offset; /* Offset into the data */ 2375 2375 u64 offset64; /* 64-bit offset */ 2376 2376 u32 avail; /* Number of bytes of available data */ 2377 2377 u32 t; /* A type code from the record header */ 2378 - u16 fx; /* pDest->flags value */ 2379 2378 Mem *pReg; /* PseudoTable input register */ 2380 2379 2381 2380 p2 = pOp->p2; 2382 2381 assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); 2383 2382 pDest = &aMem[pOp->p3]; 2384 2383 memAboutToChange(p, pDest); 2385 2384 assert( pOp->p1>=0 && pOp->p1<p->nCursor ); ................................................................................ 2549 2548 ** all valid. 2550 2549 */ 2551 2550 assert( p2<pC->nHdrParsed ); 2552 2551 assert( rc==SQLITE_OK ); 2553 2552 assert( sqlite3VdbeCheckMemInvariants(pDest) ); 2554 2553 if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); 2555 2554 assert( t==pC->aType[p2] ); 2555 + pDest->enc = encoding; 2556 2556 if( pC->szRow>=aOffset[p2+1] ){ 2557 2557 /* This is the common case where the desired content fits on the original 2558 2558 ** page - where the content is not on an overflow page */ 2559 - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); 2559 + zData = pC->aRow + aOffset[p2]; 2560 + if( t<12 ){ 2561 + sqlite3VdbeSerialGet(zData, t, pDest); 2562 + }else{ 2563 + /* If the column value is a string, we need a persistent value, not 2564 + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent 2565 + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). 2566 + */ 2567 + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; 2568 + pDest->n = len = (t-12)/2; 2569 + if( pDest->szMalloc < len+2 ){ 2570 + pDest->flags = MEM_Null; 2571 + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; 2572 + }else{ 2573 + pDest->z = pDest->zMalloc; 2574 + } 2575 + memcpy(pDest->z, zData, len); 2576 + pDest->z[len] = 0; 2577 + pDest->z[len+1] = 0; 2578 + pDest->flags = aFlag[t&1]; 2579 + } 2560 2580 }else{ 2561 2581 /* This branch happens only when content is on overflow pages */ 2562 2582 if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 2563 2583 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) 2564 2584 || (len = sqlite3VdbeSerialTypeLen(t))==0 2565 2585 ){ 2566 2586 /* Content is irrelevant for 2567 2587 ** 1. the typeof() function, 2568 2588 ** 2. the length(X) function if X is a blob, and 2569 2589 ** 3. if the content length is zero. 2570 2590 ** So we might as well use bogus content rather than reading 2571 - ** content from disk. NULL will work for the value for strings 2572 - ** and blobs and whatever is in the payloadSize64 variable 2573 - ** will work for everything else. */ 2574 - sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); 2591 + ** content from disk. */ 2592 + static u8 aZero[8]; /* This is the bogus content */ 2593 + sqlite3VdbeSerialGet(aZero, t, pDest); 2575 2594 }else{ 2576 2595 rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, 2577 2596 pDest); 2578 - if( rc!=SQLITE_OK ){ 2579 - goto op_column_error; 2597 + if( rc==SQLITE_OK ){ 2598 + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); 2599 + pDest->flags &= ~MEM_Ephem; 2580 2600 } 2581 - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); 2582 - pDest->flags &= ~MEM_Ephem; 2583 2601 } 2584 2602 } 2585 - pDest->enc = encoding; 2586 2603 2587 2604 op_column_out: 2588 - /* If the column value is an ephemeral string, go ahead and persist 2589 - ** that string in case the cursor moves before the column value is 2590 - ** used. The following code does the equivalent of Deephemeralize() 2591 - ** but does it faster. */ 2592 - if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ 2593 - fx = pDest->flags & (MEM_Str|MEM_Blob); 2594 - assert( fx!=0 ); 2595 - zData = (const u8*)pDest->z; 2596 - len = pDest->n; 2597 - if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; 2598 - memcpy(pDest->z, zData, len); 2599 - pDest->z[len] = 0; 2600 - pDest->z[len+1] = 0; 2601 - pDest->flags = fx|MEM_Term; 2602 - } 2603 2605 op_column_error: 2604 2606 UPDATE_MAX_BLOBSIZE(pDest); 2605 2607 REGISTER_TRACE(pOp->p3, pDest); 2606 2608 break; 2607 2609 } 2608 2610 2609 2611 /* Opcode: Affinity P1 P2 * P4 *
Changes to src/vdbeaux.c.
301 301 ** as having been used. 302 302 ** 303 303 ** The zWhere string must have been obtained from sqlite3_malloc(). 304 304 ** This routine will take ownership of the allocated memory. 305 305 */ 306 306 void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ 307 307 int j; 308 - int addr = sqlite3VdbeAddOp3(p, OP_ParseSchema, iDb, 0, 0); 309 - sqlite3VdbeChangeP4(p, addr, zWhere, P4_DYNAMIC); 308 + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); 310 309 for(j=0; j<p->db->nDb; j++) sqlite3VdbeUsesBtree(p, j); 311 310 } 312 311 313 312 /* 314 313 ** Add an opcode that includes the p4 value as an integer. 315 314 */ 316 315 int sqlite3VdbeAddOp4Int( ................................................................................ 801 800 ** opcodes contained within. If aOp is not NULL it is assumed to contain 802 801 ** nOp entries. 803 802 */ 804 803 static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ 805 804 if( aOp ){ 806 805 Op *pOp; 807 806 for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ 808 - freeP4(db, pOp->p4type, pOp->p4.p); 807 + if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); 809 808 #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS 810 809 sqlite3DbFree(db, pOp->zComment); 811 810 #endif 812 811 } 813 812 } 814 813 sqlite3DbFree(db, aOp); 815 814 } ................................................................................ 863 862 ** 864 863 ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points 865 864 ** to a string or structure that is guaranteed to exist for the lifetime of 866 865 ** the Vdbe. In these cases we can just copy the pointer. 867 866 ** 868 867 ** If addr<0 then change P4 on the most recently inserted instruction. 869 868 */ 869 +static void SQLITE_NOINLINE vdbeChangeP4Full( 870 + Vdbe *p, 871 + Op *pOp, 872 + const char *zP4, 873 + int n 874 +){ 875 + if( pOp->p4type ){ 876 + freeP4(p->db, pOp->p4type, pOp->p4.p); 877 + pOp->p4type = 0; 878 + pOp->p4.p = 0; 879 + } 880 + if( n<0 ){ 881 + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); 882 + }else{ 883 + if( n==0 ) n = sqlite3Strlen30(zP4); 884 + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); 885 + pOp->p4type = P4_DYNAMIC; 886 + } 887 +} 870 888 void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ 871 889 Op *pOp; 872 890 sqlite3 *db; 873 891 assert( p!=0 ); 874 892 db = p->db; 875 893 assert( p->magic==VDBE_MAGIC_INIT ); 876 - if( p->aOp==0 || db->mallocFailed ){ 877 - if( n!=P4_VTAB ){ 878 - freeP4(db, n, (void*)*(char**)&zP4); 879 - } 894 + assert( p->aOp!=0 || db->mallocFailed ); 895 + if( db->mallocFailed ){ 896 + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); 880 897 return; 881 898 } 882 899 assert( p->nOp>0 ); 883 900 assert( addr<p->nOp ); 884 901 if( addr<0 ){ 885 902 addr = p->nOp - 1; 886 903 } 887 904 pOp = &p->aOp[addr]; 888 - assert( pOp->p4type==P4_NOTUSED 889 - || pOp->p4type==P4_INT32 890 - || pOp->p4type==P4_KEYINFO ); 891 - freeP4(db, pOp->p4type, pOp->p4.p); 892 - pOp->p4.p = 0; 905 + if( n>=0 || pOp->p4type ){ 906 + vdbeChangeP4Full(p, pOp, zP4, n); 907 + return; 908 + } 893 909 if( n==P4_INT32 ){ 894 910 /* Note: this cast is safe, because the origin data point was an int 895 911 ** that was cast to a (const char *). */ 896 912 pOp->p4.i = SQLITE_PTR_TO_INT(zP4); 897 913 pOp->p4type = P4_INT32; 898 - }else if( zP4==0 ){ 899 - pOp->p4.p = 0; 900 - pOp->p4type = P4_NOTUSED; 901 - }else if( n==P4_KEYINFO ){ 902 - pOp->p4.p = (void*)zP4; 903 - pOp->p4type = P4_KEYINFO; 904 -#ifdef SQLITE_ENABLE_CURSOR_HINTS 905 - }else if( n==P4_EXPR ){ 906 - /* Responsibility for deleting the Expr tree is handed over to the 907 - ** VDBE by this operation. The caller should have already invoked 908 - ** sqlite3ExprDup() or whatever other routine is needed to make a 909 - ** private copy of the tree. */ 910 - pOp->p4.pExpr = (Expr*)zP4; 911 - pOp->p4type = P4_EXPR; 912 -#endif 913 - }else if( n==P4_VTAB ){ 914 - pOp->p4.p = (void*)zP4; 915 - pOp->p4type = P4_VTAB; 916 - sqlite3VtabLock((VTable *)zP4); 917 - assert( ((VTable *)zP4)->db==p->db ); 918 - }else if( n<0 ){ 914 + }else if( zP4!=0 ){ 915 + assert( n<0 ); 919 916 pOp->p4.p = (void*)zP4; 920 917 pOp->p4type = (signed char)n; 921 - }else{ 922 - if( n==0 ) n = sqlite3Strlen30(zP4); 923 - pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); 924 - pOp->p4type = P4_DYNAMIC; 918 + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); 925 919 } 926 920 } 927 921 928 922 /* 929 923 ** Set the P4 on the most recently added opcode to the KeyInfo for the 930 924 ** index given. 931 925 */
Changes to src/vdbesort.c.
733 733 assert( pFile->iEof>iStart ); 734 734 assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); 735 735 assert( pReadr->aBuffer==0 ); 736 736 assert( pReadr->aMap==0 ); 737 737 738 738 rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); 739 739 if( rc==SQLITE_OK ){ 740 - u64 nByte; /* Size of PMA in bytes */ 740 + u64 nByte = 0; /* Size of PMA in bytes */ 741 741 rc = vdbePmaReadVarint(pReadr, &nByte); 742 742 pReadr->iEof = pReadr->iReadOff + nByte; 743 743 *pnByte += nByte; 744 744 } 745 745 746 746 if( rc==SQLITE_OK ){ 747 747 rc = vdbePmaReaderNext(pReadr);
Changes to src/vxworks.h.
22 22 #define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 23 23 #define SQLITE_OMIT_LOAD_EXTENSION 1 24 24 #define SQLITE_ENABLE_LOCKING_STYLE 0 25 25 #define HAVE_UTIME 1 26 26 #else 27 27 /* This is not VxWorks. */ 28 28 #define OS_VXWORKS 0 29 +#define HAVE_FCHOWN 1 30 +#define HAVE_READLINK 1 29 31 #endif /* defined(_WRS_KERNEL) */
Changes to src/wal.c.
704 704 705 705 nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); 706 706 walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); 707 707 walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); 708 708 709 709 sqlite3Put4byte(&aFrame[16], aCksum[0]); 710 710 sqlite3Put4byte(&aFrame[20], aCksum[1]); 711 + }else{ 712 + memset(&aFrame[8], 0, 16); 711 713 } 712 714 } 713 715 714 716 /* 715 717 ** Check to see if the frame with header in aFrame[] and content 716 718 ** in aData[] is valid. If it is a valid frame, fill *piPage and 717 719 ** *pnTruncate and return true. Return if the frame is not valid. ................................................................................ 2619 2621 Pgno sqlite3WalDbsize(Wal *pWal){ 2620 2622 if( pWal && ALWAYS(pWal->readLock>=0) ){ 2621 2623 return pWal->hdr.nPage; 2622 2624 } 2623 2625 return 0; 2624 2626 } 2625 2627 2626 -/* 2627 - ** Return the file for this Wal journal (or zero, if unknown). 2628 - */ 2629 -sqlite3_file *sqlite3WalFile(Wal *pWal){ 2630 - if( pWal ){ 2631 - return pWal->pWalFd; 2632 - } 2633 - return 0; 2634 -} 2635 - 2636 2628 /* 2637 2629 ** This function starts a write transaction on the WAL. 2638 2630 ** 2639 2631 ** A read transaction must have already been started by a prior call 2640 2632 ** to sqlite3WalBeginReadTransaction(). 2641 2633 ** 2642 2634 ** If another thread or process has written into the database since ................................................................................ 3445 3437 ** or zero if it is not (or if pWal is NULL). 3446 3438 */ 3447 3439 int sqlite3WalFramesize(Wal *pWal){ 3448 3440 assert( pWal==0 || pWal->readLock>=0 ); 3449 3441 return (pWal ? pWal->szPage : 0); 3450 3442 } 3451 3443 #endif 3444 + 3445 +/* Return the sqlite3_file object for the WAL file 3446 +*/ 3447 +sqlite3_file *sqlite3WalFile(Wal *pWal){ 3448 + return pWal->pWalFd; 3449 +} 3452 3450 3453 3451 #endif /* #ifndef SQLITE_OMIT_WAL */
Changes to src/wal.h.
40 40 # define sqlite3WalFrames(u,v,w,x,y,z) 0 41 41 # define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 42 42 # define sqlite3WalCallback(z) 0 43 43 # define sqlite3WalExclusiveMode(y,z) 0 44 44 # define sqlite3WalHeapMemory(z) 0 45 45 # define sqlite3WalFramesize(z) 0 46 46 # define sqlite3WalFindFrame(x,y,z) 0 47 +# define sqlite3WalFile(x) 0 47 48 #else 48 49 49 50 #define WAL_SAVEPOINT_NDATA 4 50 51 51 52 /* Connection to a write-ahead log (WAL) file. 52 53 ** There is one object of this type for each pager. 53 54 */ ................................................................................ 136 137 137 138 #ifdef SQLITE_ENABLE_ZIPVFS 138 139 /* If the WAL file is not empty, return the number of bytes of content 139 140 ** stored in each frame (i.e. the db page-size when the WAL was created). 140 141 */ 141 142 int sqlite3WalFramesize(Wal *pWal); 142 143 #endif 144 + 145 +/* Return the sqlite3_file object for the WAL file */ 146 +sqlite3_file *sqlite3WalFile(Wal *pWal); 143 147 144 148 #endif /* ifndef SQLITE_OMIT_WAL */ 145 149 #endif /* _WAL_H_ */
Changes to src/walker.c.
32 32 ** 33 33 ** WRC_Abort Do no more callbacks. Unwind the stack and 34 34 ** return the top-level walk call. 35 35 ** 36 36 ** The return value from this routine is WRC_Abort to abandon the tree walk 37 37 ** and WRC_Continue to continue. 38 38 */ 39 -int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ 39 +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ 40 40 int rc; 41 - if( pExpr==0 ) return WRC_Continue; 42 41 testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); 43 42 testcase( ExprHasProperty(pExpr, EP_Reduced) ); 44 43 rc = pWalker->xExprCallback(pWalker, pExpr); 45 44 if( rc==WRC_Continue 46 45 && !ExprHasProperty(pExpr,EP_TokenOnly) ){ 47 46 if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; 48 47 if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; ................................................................................ 50 49 if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; 51 50 }else{ 52 51 if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; 53 52 } 54 53 } 55 54 return rc & WRC_Abort; 56 55 } 56 +int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ 57 + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; 58 +} 57 59 58 60 /* 59 61 ** Call sqlite3WalkExpr() for every expression in list p or until 60 62 ** an abort request is seen. 61 63 */ 62 64 int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ 63 65 int i;
Changes to src/where.c.
4314 4314 assert( pTabItem->iCursor==pLevel->iTabCur ); 4315 4315 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); 4316 4316 testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); 4317 4317 if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol<BMS && HasRowid(pTab) ){ 4318 4318 Bitmask b = pTabItem->colUsed; 4319 4319 int n = 0; 4320 4320 for(; b; b=b>>1, n++){} 4321 - sqlite3VdbeChangeP4(v, sqlite3VdbeCurrentAddr(v)-1, 4322 - SQLITE_INT_TO_PTR(n), P4_INT32); 4321 + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); 4323 4322 assert( n<=pTab->nCol ); 4324 4323 } 4325 4324 #ifdef SQLITE_ENABLE_CURSOR_HINTS 4326 4325 if( pLoop->u.btree.pIndex!=0 ){ 4327 4326 sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); 4328 4327 }else 4329 4328 #endif
Changes to test/orderby1.test.
523 523 INSERT INTO t1 SELECT i%2, randomblob(500) FROM cnt; 524 524 } 525 525 526 526 do_test 8.3 { 527 527 db eval { SELECT * FROM t1 ORDER BY a, b } { incr res $a } 528 528 set res 529 529 } 5000 530 + 531 +#--------------------------------------------------------------------------- 532 +# https://www.sqlite.org/src/tktview/cb3aa0641d9a413841c004293a4fc06cdc122029 533 +# 534 +# Adverse interaction between scalar subqueries and the partial-sorting 535 +# logic. 536 +# 537 +do_execsql_test 9.0 { 538 + DROP TABLE IF EXISTS t1; 539 + CREATE TABLE t1(x INTEGER PRIMARY KEY); 540 + INSERT INTO t1 VALUES(1),(2); 541 + DROP TABLE IF EXISTS t2; 542 + CREATE TABLE t2(y); 543 + INSERT INTO t2 VALUES(9),(8),(3),(4); 544 + SELECT (SELECT x||y FROM t2, t1 ORDER BY x, y); 545 +} {13} 546 + 530 547 531 548 finish_test
Changes to test/waloverwrite.test.
150 150 do_test 1.$tn.9 { 151 151 db2 close 152 152 forcecopy test.db-wal test.db2-wal 153 153 sqlite3 db2 test.db2 154 154 execsql { SELECT sum(length(y)) FROM t1 } db2 155 155 } [expr 20*798] 156 156 157 - do_test 1.$tn.9 { 157 + do_test 1.$tn.10 { 158 158 execsql { PRAGMA integrity_check } db2 159 159 } ok 160 160 db2 close 161 161 } 162 162 163 163 finish_test 164 164
Changes to tool/sqldiff.c.
990 990 if( lenSrc<=NHASH ){ 991 991 putInt(lenOut, &zDelta); 992 992 *(zDelta++) = ':'; 993 993 memcpy(zDelta, zOut, lenOut); 994 994 zDelta += lenOut; 995 995 putInt(checksum(zOut, lenOut), &zDelta); 996 996 *(zDelta++) = ';'; 997 - return zDelta - zOrigDelta; 997 + return (int)(zDelta - zOrigDelta); 998 998 } 999 999 1000 1000 /* Compute the hash table used to locate matching sections in the 1001 1001 ** source file. 1002 1002 */ 1003 1003 nHash = lenSrc/NHASH; 1004 1004 collide = sqlite3_malloc( nHash*2*sizeof(int) ); ................................................................................ 1137 1137 memcpy(zDelta, &zOut[base], lenOut-base); 1138 1138 zDelta += lenOut-base; 1139 1139 } 1140 1140 /* Output the final checksum record. */ 1141 1141 putInt(checksum(zOut, lenOut), &zDelta); 1142 1142 *(zDelta++) = ';'; 1143 1143 sqlite3_free(collide); 1144 - return zDelta - zOrigDelta; 1144 + return (int)(zDelta - zOrigDelta); 1145 1145 } 1146 1146 1147 1147 /* 1148 1148 ** End of code copied from fossil. 1149 1149 **************************************************************************/ 1150 1150 1151 1151 static void strPrintfArray(