Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts3-prefix-search |
Files: | files | file ages | folders |
SHA1: |
be59bf49402d2e2f4b95fb6668849f37 |
User & Date: | dan 2011-05-25 18:34:53.807 |
Context
2011-05-25
| ||
18:47 | Merge trunk changes into experimental fts3-prefix-search branch. (check-in: f0f0a03db2 user: dan tags: fts3-prefix-search) | |
18:34 | Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. (check-in: be59bf4940 user: dan tags: fts3-prefix-search) | |
2011-05-24
| ||
18:49 | If the fts4 option prefix=1 is specified, have the fts4 module maintain an index of prefixes as well as terms. (check-in: b5bdc63989 user: dan tags: fts3-prefix-search) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
827 828 829 830 831 832 833 834 835 836 837 838 839 840 | fts3Appendf(pRc, &zRet, "?"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } sqlite3_free(zFree); return zRet; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 | fts3Appendf(pRc, &zRet, "?"); for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } sqlite3_free(zFree); return zRet; } static int fts3GobbleInt(const char **pp, int *pnOut){ const char *p = *pp; int nInt = 0; for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; *pp = p; return SQLITE_OK; } static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ ){ struct Fts3Index *aIndex; int nIndex = 1; if( zParam && zParam[0] ){ const char *p; nIndex++; for(p=zParam; *p; p++){ if( *p==',' ) nIndex++; } } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); *apIndex = *apFree = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; } memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); if( zParam ){ const char *p = zParam; int i; for(i=1; i<nIndex; i++){ int nPrefix; if( fts3GobbleInt(&p, &nPrefix) ) return SQLITE_ERROR; aIndex[i].nPrefix = nPrefix; p++; } } return SQLITE_OK; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** |
︙ | ︙ | |||
861 862 863 864 865 866 867 | int nString = 0; /* Bytes required to hold all column names */ int nCol = 0; /* Number of columns in the FTS table */ char *zCsr; /* Space for holding column names */ int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ | < > > > > > | 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 | int nString = 0; /* Bytes required to hold all column names */ int nCol = 0; /* Number of columns in the FTS table */ char *zCsr; /* Space for holding column names */ int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ int nIndex; /* Size of aIndex[] array */ struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ char *zCompress = 0; char *zUncompress = 0; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) |
︙ | ︙ | |||
925 926 927 928 929 930 931 | }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ zCompress = zVal; zVal = 0; }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ | > | > | 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 | }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ zCompress = zVal; zVal = 0; }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ sqlite3_free(zPrefix); zPrefix = zVal; zVal = 0; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); } |
︙ | ︙ | |||
954 955 956 957 958 959 960 | if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; } assert( pTokenizer ); | > > > > | > > > | > | | > > > | | > | | 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 | if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; } assert( pTokenizer ); rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); if( rc==SQLITE_ERROR ){ assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); } if( rc ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ nIndex * sizeof(struct Fts3Index) + /* aIndex */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ p = (Fts3Table*)sqlite3_malloc(nByte); if( p==0 ){ rc = SQLITE_NOMEM; goto fts3_init_out; } memset(p, 0, nByte); p->db = db; p->nColumn = nCol; p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); p->nIndex = nIndex; for(i=0; i<nIndex; i++){ fts3HashInit(&p->aIndex[i].hPending, FTS3_HASH_STRING, 1); } /* Fill in the zName and zDb fields of the vtab structure. */ zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; p->zDb = zCsr; memcpy(zCsr, argv[1], nDb); zCsr += nDb; |
︙ | ︙ | |||
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 | */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ | > > | 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 | */ fts3DatabasePageSize(&rc, p); /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zPrefix); sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ |
︙ | ︙ | |||
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } static int fts3SegReaderCursorAppend( Fts3SegReaderCursor *pCsr, Fts3SegReader *pNew ){ if( (pCsr->nSegment%16)==0 ){ Fts3SegReader **apNew; int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); if( !apNew ){ sqlite3Fts3SegReaderFree(pNew); return SQLITE_NOMEM; } pCsr->apSegment = apNew; } pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } /* | > > > | | > > | < < | | < < < | < < < < < | | < | | > | > > > > > | < < < < | < | | | | | > | 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 | if( !*ppOut ) return SQLITE_NOMEM; sqlite3Fts3PutVarint(*ppOut, docid); } return SQLITE_OK; } /* ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ static int fts3SegReaderCursorAppend( Fts3SegReaderCursor *pCsr, Fts3SegReader *pNew ){ if( (pCsr->nSegment%16)==0 ){ Fts3SegReader **apNew; int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); if( !apNew ){ sqlite3Fts3SegReaderFree(pNew); return SQLITE_NOMEM; } pCsr->apSegment = apNew; } pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } /* ** Set up a cursor object for iterating through a full-text index or a ** single level therein. */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ Fts3SegReaderCursor *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; int iAge = 0; sqlite3_stmt *pStmt = 0; assert( iIndex>=0 && iIndex<p->nIndex ); assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( FTS3_SEGCURSOR_ALL<0 && FTS3_SEGCURSOR_PENDING<0 ); assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); assert( isPrefix==0 || isScan==0 ); /* "isScan" is only set to true by the ft4aux module, an ordinary ** full-text tables. */ assert( isScan==0 || p->aIndex==0 ); memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling ** Fts3SegReaderPending might segfault, as the data structures used by ** fts4aux are not completely populated. So it's easiest to filter these ** calls out here. */ if( iLevel<0 && p->aIndex ){ Fts3SegReader *pSeg = 0; rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); if( rc==SQLITE_OK && pSeg ){ rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); |
︙ | ︙ | |||
2269 2270 2271 2272 2273 2274 2275 2276 2277 | Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ int i; int nCost = 0; Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; | > | > > > | | > | > > | > | > < | 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 | Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ int i; int nCost = 0; int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; if( isPrefix ){ for(i=1; i<p->nIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; rc = sqlite3Fts3SegReaderCursor( p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); break; } } } if( bFound==0 ){ rc = sqlite3Fts3SegReaderCursor( p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); } for(i=0; rc==SQLITE_OK && i<pSegcsr->nSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); } pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; |
︙ | ︙ | |||
3336 3337 3338 3339 3340 3341 3342 | */ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; sqlite3Fts3PendingTermsClear(p); assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); | < | 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 | */ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; sqlite3Fts3PendingTermsClear(p); assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; } /* ** Load the doclist associated with expression pExpr to pExpr->aDoclist. ** The loaded doclist contains positions as well as the document ids. ** This is used by the matchinfo(), snippet() and offsets() auxillary |
︙ | ︙ | |||
3711 3712 3713 3714 3715 3716 3717 | } static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); | < | 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 | } static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); return SQLITE_OK; } static const sqlite3_module fts3Module = { /* iVersion */ 2, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
19 20 21 22 23 24 25 | # define NDEBUG 1 #endif #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" | < < | 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | # define NDEBUG 1 #endif #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single ** segment of level N+1. */ #define FTS3_MERGE_COUNT 16 |
︙ | ︙ | |||
52 53 54 55 56 57 58 | /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 /* | | < | | | > > > > > | | | | | | 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. */ #define FTS3_VARINT_MAX 10 /* ** FTS4 virtual tables may maintain multiple indexes - one index of all terms ** in the document set and zero or more prefix indexes. All indexes are stored ** as one or more b+-trees in the %_segments and %_segdir tables. ** ** It is possible to determine which index a b+-tree belongs to based on the ** value stored in the "%_segdir.level" column. Given this value L, the index ** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with ** level values between 0 and 1023 (inclusive) belong to index 0, all levels ** between 1024 and 2047 to index 1, and so on. ** ** It is considered impossible for an index to use more than 1024 levels. In ** theory though this may happen, but only after at least ** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ #define FTS3_SEGDIR_MAXLEVEL 1024 #define FTS3_SEGDIR_MAXLEVEL_STR "1024" /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. */ #ifndef testcase # define testcase(X) |
︙ | ︙ | |||
168 169 170 171 172 173 174 | char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ | < > > | > > > > > > > > > > | | | < < | 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* TODO: Fix the first paragraph of this comment. ** ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. ** ** A single FTS4 table may have multiple full-text indexes. For each index ** there is an entry in the aIndex[] array. Index 0 is an index of all the ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. */ int nIndex; /* Size of aIndex[] */ struct Fts3Index { int nPrefix; /* Prefix length (0 for main terms index) */ Fts3Hash hPending; /* Pending terms table for this index */ } *aIndex; int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These ** values do not contribution to the FTS computation; they are used for ** verifying the SQLite core. */ |
︙ | ︙ | |||
332 333 334 335 336 337 338 | int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending( | | | < | < | | 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 | int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); int sqlite3Fts3PendingTermsFlush(Fts3Table *); void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 #define FTS3_SEGMENT_SCAN 0x00000010 |
︙ | ︙ |
Changes to ext/fts3/fts3_aux.c.
︙ | ︙ | |||
92 93 94 95 96 97 98 99 100 101 102 103 104 105 | if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); p->pFts3Tab = (Fts3Table *)&p[1]; p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; | > | 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 | if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); p->pFts3Tab = (Fts3Table *)&p[1]; p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; |
︙ | ︙ | |||
371 372 373 374 375 376 377 | if( idxNum&FTS4AUX_LE_CONSTRAINT ){ int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); if( pCsr->zStop==0 ) return SQLITE_NOMEM; } | | | 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 | if( idxNum&FTS4AUX_LE_CONSTRAINT ){ int iIdx = (idxNum&FTS4AUX_GE_CONSTRAINT) ? 1 : 0; pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iIdx])); pCsr->nStop = sqlite3_value_bytes(apVal[iIdx]); if( pCsr->zStop==0 ) return SQLITE_NOMEM; } rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); } if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); |
︙ | ︙ |
Changes to ext/fts3/fts3_term.c.
︙ | ︙ | |||
23 24 25 26 27 28 29 | #include <assert.h> typedef struct Fts3termTable Fts3termTable; typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ | | | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 | #include <assert.h> typedef struct Fts3termTable Fts3termTable; typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ int iIndex; /* Index for Fts3Table.aIndex[] */ Fts3Table *pFts3Tab; }; struct Fts3termCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ Fts3SegReaderCursor csr; /* Must be right after "base" */ Fts3SegFilter filter; |
︙ | ︙ | |||
66 67 68 69 70 71 72 73 74 75 76 77 78 79 | char const *zDb; /* Name of database (e.g. "main") */ char const *zFts3; /* Name of fts3 table */ int nDb; /* Result of strlen(zDb) */ int nFts3; /* Result of strlen(zFts3) */ int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ *pzErr = sqlite3_mprintf( "wrong number of arguments to fts4term constructor" ); return SQLITE_ERROR; | > > > > > > | 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | char const *zDb; /* Name of database (e.g. "main") */ char const *zFts3; /* Name of fts3 table */ int nDb; /* Result of strlen(zDb) */ int nFts3; /* Result of strlen(zFts3) */ int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ int iIndex = 0; if( argc==5 ){ iIndex = atoi(argv[4]); argc--; } /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ *pzErr = sqlite3_mprintf( "wrong number of arguments to fts4term constructor" ); return SQLITE_ERROR; |
︙ | ︙ | |||
92 93 94 95 96 97 98 | if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); p->pFts3Tab = (Fts3Table *)&p[1]; p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; | > | | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 | if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); p->pFts3Tab = (Fts3Table *)&p[1]; p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; p->pFts3Tab->nIndex = iIndex+1; p->iIndex = iIndex; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; |
︙ | ︙ | |||
259 260 261 262 263 264 265 | testcase(pCsr->filter.zTerm); sqlite3Fts3SegReaderFinish(&pCsr->csr); memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; | | < | 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 | testcase(pCsr->filter.zTerm); sqlite3Fts3SegReaderFinish(&pCsr->csr); memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); } if( rc==SQLITE_OK ){ rc = fts3termNextMethod(pCursor); |
︙ | ︙ | |||
351 352 353 354 355 356 357 | 0, /* xRollback */ 0, /* xFindFunction */ 0 /* xRename */ }; int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); | < < < | 357 358 359 360 361 362 363 364 365 366 367 368 | 0, /* xRollback */ 0, /* xFindFunction */ 0 /* xRename */ }; int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); return rc; } #endif #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
179 180 181 182 183 184 185 | #define SQL_DELETE_ALL_STAT 6 #define SQL_SELECT_CONTENT_BY_ROWID 7 #define SQL_NEXT_SEGMENT_INDEX 8 #define SQL_INSERT_SEGMENTS 9 #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 | | | > > | < | 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 | #define SQL_DELETE_ALL_STAT 6 #define SQL_SELECT_CONTENT_BY_ROWID 7 #define SQL_NEXT_SEGMENT_INDEX 8 #define SQL_INSERT_SEGMENTS 9 #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 #define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 #define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 #define SQL_DELETE_SEGDIR_RANGE 26 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** |
︙ | ︙ | |||
230 231 232 233 234 235 236 | /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " | | | | < | | | < | > | 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 | /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", /* 24 */ "", /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
︙ | ︙ | |||
405 406 407 408 409 410 411 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ | | > > > > > > | > > | | > | > > > > < > | > > | 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 | ** ** 0: idx ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ int sqlite3Fts3AllSegdirs( Fts3Table *p, /* FTS3 table */ int iIndex, /* Index for p->aIndex[] */ int iLevel, /* Level to select */ sqlite3_stmt **ppStmt /* OUT: Compiled statement */ ){ int rc; sqlite3_stmt *pStmt = 0; assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( iIndex>=0 && iIndex<p->nIndex ); if( iLevel==FTS3_SEGCURSOR_ALL ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); } } *ppStmt = pStmt; return rc; } /* |
︙ | ︙ | |||
560 561 562 563 564 565 566 | } } if( rc==SQLITE_OK ){ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } | < < < | 575 576 577 578 579 580 581 582 583 584 585 586 587 588 | } } if( rc==SQLITE_OK ){ p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); } return rc; } /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in ** p->iPrevDocid, and the column is specified by argument iCol. ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. |
︙ | ︙ | |||
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 | } pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. ** Tokens must have a non-zero length. */ if( iPos<0 || !zToken || nToken<=0 ){ rc = SQLITE_ERROR; break; } | > > | > | | > > | > > | < > > | 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 | } pCsr->pTokenizer = pTokenizer; xNext = pModule->xNext; while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. ** Tokens must have a non-zero length. */ if( iPos<0 || !zToken || nToken<=0 ){ rc = SQLITE_ERROR; break; } /* Add the term to the terms index */ rc = fts3PendingTermsAddOne( p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken ); /* Add the term to each of the prefix indexes that it is not too ** short for. */ for(i=1; rc==SQLITE_OK && i<p->nIndex; i++){ struct Fts3Index *pIndex = &p->aIndex[i]; if( nToken<pIndex->nPrefix ) continue; rc = fts3PendingTermsAddOne( p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix ); } } pModule->xClose(pCsr); *pnWord = nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } |
︙ | ︙ | |||
657 658 659 660 661 662 663 | if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; } /* | | > > | > | | | | < | | < < < < < < < < < | 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 | if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; return SQLITE_OK; } /* ** Discard the contents of the pending-terms hash tables. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ int i; for(i=0; i<p->nIndex; i++){ Fts3HashElem *pElem; Fts3Hash *pHash = &p->aIndex[i].hPending; for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ sqlite3_free(fts3HashData(pElem)); } fts3HashClear(pHash); } p->nPendingData = 0; } /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the ** pendingTerms hash table. ** |
︙ | ︙ | |||
776 777 778 779 780 781 782 | ** pending terms. */ static int fts3DeleteAll(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); | < | 789 790 791 792 793 794 795 796 797 798 799 800 801 802 | ** pending terms. */ static int fts3DeleteAll(Fts3Table *p){ int rc = SQLITE_OK; /* Return code */ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); |
︙ | ︙ | |||
832 833 834 835 836 837 838 | *pRC = rc; } /* ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ | | | > > > > > | | | 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 | *pRC = rc; } /* ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. ** Usually, indexes are allocated within a level sequentially starting ** with 0, so the allocated index is one greater than the value returned ** by: ** ** SELECT max(idx) FROM %_segdir WHERE level = :iLevel ** ** However, if there are already FTS3_MERGE_COUNT indexes at the requested ** level, they are merged into a single level (iLevel+1) segment and the ** allocated index is 0. ** ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ static int fts3AllocateSegdirIdx( Fts3Table *p, int iIndex, /* Index for p->aIndex */ int iLevel, int *piIdx ){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } rc = sqlite3_reset(pNextIdx); } if( rc==SQLITE_OK ){ /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already ** full, merge all segments in level iLevel into a single iLevel+1 ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; } } return rc; |
︙ | ︙ | |||
1305 1306 1307 1308 1309 1310 1311 1312 1313 | ** ** Whereas if isPrefixIter is zero, the terms visited are: ** ** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ | > | < > > | < < < | 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 | ** ** Whereas if isPrefixIter is zero, the terms visited are: ** ** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ Fts3Hash *pHash; pHash = &p->aIndex[iIndex].hPending; if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ if( nElem==nAlloc ){ Fts3HashElem **aElem2; |
︙ | ︙ | |||
1356 1357 1358 1359 1360 1361 1362 | if( nElem>1 ){ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); } }else{ /* The query is a simple term lookup that matches at most one term in ** the index. All that is required is a straight hash-lookup. */ | | | | 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 | if( nElem>1 ){ qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); } }else{ /* The query is a simple term lookup that matches at most one term in ** the index. All that is required is a straight hash-lookup. */ Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; } } if( nElem>0 ){ int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); pReader = (Fts3SegReader *)sqlite3_malloc(nByte); if( !pReader ){ rc = SQLITE_NOMEM; }else{ memset(pReader, 0, nByte); pReader->iIdx = 0x7FFFFFFF; pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); } } if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; return rc; } /* |
︙ | ︙ | |||
1988 1989 1990 1991 1992 1993 1994 | } rc = sqlite3_reset(pStmt); } return rc; } /* | | < | | | | | > | 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 | } rc = sqlite3_reset(pStmt); } return rc; } /* ** Set *pnMax to the largest segment level in the database for the index ** iIndex. ** ** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; assert( iIndex>=0 && iIndex<p->nIndex ); /* Set pStmt to the compiled version of: ** ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } /* |
︙ | ︙ | |||
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 | ** 2) deletes all %_segdir entries with level iLevel, or all %_segdir ** entries regardless of level if (iLevel<0). ** ** SQLITE_OK is returned if successful, otherwise an SQLite error code. */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ int rc; /* Return Code */ int i; /* Iterator variable */ sqlite3_stmt *pDelete; /* SQL statement to delete rows */ | > | 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 | ** 2) deletes all %_segdir entries with level iLevel, or all %_segdir ** entries regardless of level if (iLevel<0). ** ** SQLITE_OK is returned if successful, otherwise an SQLite error code. */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ ){ int rc; /* Return Code */ int i; /* Iterator variable */ sqlite3_stmt *pDelete; /* SQL statement to delete rows */ |
︙ | ︙ | |||
2054 2055 2056 2057 2058 2059 2060 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } | | < < < < < | | | < < < | | > | | | > > > > | | | < | 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 | rc = sqlite3_reset(pDelete); } } if( rc!=SQLITE_OK ){ return rc; } assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); if( iLevel==FTS3_SEGCURSOR_ALL ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } }else{ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); } } if( rc==SQLITE_OK ){ sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } return rc; } /* ** When this function is called, buffer *ppList (size *pnList bytes) contains ** a position list that may (or may not) feature multiple columns. This |
︙ | ︙ | |||
2319 2320 2321 2322 2323 2324 2325 | ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ | | | > > > > > > > | | | | | > > > > < < < < < < < | > | > > | | > | | < < | | < | < > | 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 | ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( iIndex>=0 && iIndex<p->nIndex ); rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); bIgnoreEmpty = 1; }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ rc = sqlite3Fts3SegReaderStep(p, &csr); if( rc!=SQLITE_ROW ) break; rc = fts3SegWriterAdd(p, &pWriter, 1, csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); } if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); if( rc!=SQLITE_OK ) goto finished; } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); return rc; } /* ** Encode N integers as varints into a blob. */ static void fts3EncodeIntArray( |
︙ | ︙ | |||
2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 | return; } sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); } /* ** Handle a 'special' INSERT of the form: ** ** "INSERT INTO tbl(tbl) VALUES(<expr>)" ** ** Argument pVal contains the result of <expr>. Currently the only ** meaningful value to insert is the text 'optimize'. */ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ | > > > > > > > > > > > > > > > > > < < | < < < < | 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 | return; } sqlite3_bind_blob(pStmt, 1, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); } static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ int i; int bSeenDone = 0; int rc = SQLITE_OK; for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_DONE ){ bSeenDone = 1; rc = SQLITE_OK; } } sqlite3Fts3SegmentsClose(p); sqlite3Fts3PendingTermsClear(p); return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } /* ** Handle a 'special' INSERT of the form: ** ** "INSERT INTO tbl(tbl) VALUES(<expr>)" ** ** Argument pVal contains the result of <expr>. Currently the only ** meaningful value to insert is the text 'optimize'. */ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ int rc; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); rc = SQLITE_OK; }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ p->nMaxPendingData = atoi(&zVal[11]); rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; } return rc; } /* ** Return the deferred doclist associated with deferred token pDeferred. ** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already ** been called to allocate and populate the doclist. |
︙ | ︙ | |||
2901 2902 2903 2904 2905 2906 2907 | /* ** Flush any data in the pending-terms hash table to disk. If successful, ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; | < < < < < < < | < | | | < < < | | 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 | /* ** Flush any data in the pending-terms hash table to disk. If successful, ** merge all segments in the database (including the new segment, if ** there was any data to flush) into a single segment. */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ rc = fts3DoOptimize(p, 1); if( rc==SQLITE_OK || rc==SQLITE_DONE ){ int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } sqlite3Fts3SegmentsClose(p); return rc; } #endif |
Changes to test/fts3prefix.test.
︙ | ︙ | |||
11 12 13 14 15 16 17 18 19 20 21 | # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts3prefix # This proc tests that the prefixes index appears to represent the same content # as the terms index. # | > > > > > | > > > > > > > > | | | > > | | | | | | > | | < | | | | > > > | > | | | < < | > | | > | | < < > | | | > | | | > | > > | | | | | 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS3 module. # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts3prefix ifcapable !fts3 { finish_test return } # This proc tests that the prefixes index appears to represent the same content # as the terms index. # proc fts3_terms_and_prefixes {db tbl prefixlengths} { set iIndex 0 foreach len $prefixlengths { incr iIndex $db eval { DROP TABLE IF EXISTS fts3check1; DROP TABLE IF EXISTS fts3check2; } $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" $db eval { DROP TABLE IF EXISTS temp.terms; DROP TABLE IF EXISTS temp.prefixes; CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; CREATE INDEX temp.idx ON prefixes(term); DROP TABLE fts3check1; DROP TABLE fts3check2; } set nExpect 0 $db eval { SELECT term, docid, col, pos FROM temp.terms } a { if {[string length $a(term)]<$len} continue incr nExpect set prefix [string range $a(term) 0 [expr $len-1]] set r [$db one { SELECT count(*) FROM temp.prefixes WHERE term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) }] if {$r != 1} { error "$t, $a(docid), $a(col), $a(pos)" } } set nCount [$db one {SELECT count(*) FROM temp.prefixes}] if {$nCount != $nExpect} { error "prefixes.count(*) is $nCount expected $nExpect" } execsql { DROP TABLE temp.prefixes } execsql { DROP TABLE temp.terms } set list [list] $db eval " SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree FROM ${tbl}_segdir GROUP BY tree " { lappend list [list $total $tree] } if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { error "inconsistent tree structures: $list" } } return "" } proc fts3_tap_test {tn db tbl lens} { uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] } #------------------------------------------------------------------------- # Test cases 1.* are a sanity check. They test that the prefixes index is # being constructed correctly for the simplest possible case. # do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); CREATE VIRTUAL TABLE terms USING fts4term(t1); } do_execsql_test 1.2 { INSERT INTO t1 VALUES('sqlite mysql firebird'); } do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} do_execsql_test 1.4 { SELECT term FROM terms; } {firebird mysql sqlite} fts3_tap_test 1.5 db t1 {1 3 6} #------------------------------------------------------------------------- # A slightly more complicated dataset. This test also verifies that DELETE # operations do not corrupt the prefixes index. # do_execsql_test 2.1 { INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); |
︙ | ︙ | |||
111 112 113 114 115 116 117 | INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); INSERT INTO t1 VALUES('these older modules and their use should be'); INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); INSERT INTO t1 VALUES('of SQLite. '); } | | | | | | 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 | INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); INSERT INTO t1 VALUES('these older modules and their use should be'); INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); INSERT INTO t1 VALUES('of SQLite. '); } fts3_tap_test 2.2 db t1 {1 3 6} do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } fts3_tap_test 2.4 db t1 {1 3 6} do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } fts3_tap_test 2.6 db t1 {1 3 6} do_execsql_test 3.1 { CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); INSERT INTO t2 VALUES('fleet was forced to take shelter at'); INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); |
︙ | ︙ | |||
144 145 146 147 148 149 150 | INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); } | | | | | 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 | INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); } fts3_tap_test 3.2 db t2 {1 2 3} do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} fts3_tap_test 3.4 db t2 {1 2 3} #------------------------------------------------------------------------- # Simple tests for reading the prefix-index. # do_execsql_test 4.1 { CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); INSERT INTO t3 VALUES('one two three'); INSERT INTO t3 VALUES('four five six'); INSERT INTO t3 VALUES('seven eight nine'); } do_execsql_test 4.2 { SELECT * FROM t3 WHERE t3 MATCH 'f*' } {{four five six}} |
︙ | ︙ |