Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add a cookie mechanism to ensure that the %_config table is re-read as required. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
bb4a37b53de60da9ec8b9317eec14afa |
User & Date: | dan 2014-11-28 20:01:13.778 |
Context
2014-12-01
| ||
20:05 | Add code to parse a rank() function specification. And a tcl interface to add auxiliary functions to fts5. (check-in: 9c1697a2aa user: dan tags: fts5) | |
2014-11-28
| ||
20:01 | Add a cookie mechanism to ensure that the %_config table is re-read as required. (check-in: bb4a37b53d user: dan tags: fts5) | |
2014-11-27
| ||
20:03 | Add a %_config table to fts5. (check-in: 83491c5666 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
336 337 338 339 340 341 342 | } /* Call sqlite3_declare_vtab() */ if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigDeclareVtab(pConfig); } | < < < < < | 336 337 338 339 340 341 342 343 344 345 346 347 348 349 | } /* Call sqlite3_declare_vtab() */ if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigDeclareVtab(pConfig); } if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab, 0); pTab = 0; }else if( bCreate ){ fts5CheckTransactionState(pTab, FTS5_BEGIN, 0); } *ppVTab = (sqlite3_vtab*)pTab; |
︙ | ︙ |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | typedef struct Fts5Config Fts5Config; /* ** An instance of the following structure encodes all information that can ** be gleaned from the CREATE VIRTUAL TABLE statement. ** ** And all information loaded from the %_config table. */ struct Fts5Config { sqlite3 *db; /* Database handle */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ char **azCol; /* Column names */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ }; int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); /* Load the contents of the %_config table */ | > > > > > > | | 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 | typedef struct Fts5Config Fts5Config; /* ** An instance of the following structure encodes all information that can ** be gleaned from the CREATE VIRTUAL TABLE statement. ** ** And all information loaded from the %_config table. ** ** nAutomerge: ** The minimum number of segments that an auto-merge operation should ** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. */ struct Fts5Config { sqlite3 *db; /* Database handle */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ char **azCol; /* Column names */ int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ Fts5Tokenizer *pTok; fts5_tokenizer *pTokApi; /* Values loaded from the %_config table */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ }; int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** ); void sqlite3Fts5ConfigFree(Fts5Config*); int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig); int sqlite3Fts5Tokenize( Fts5Config *pConfig, /* FTS5 Configuration object */ const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, const char*, int, int, int, int) /* Callback */ ); void sqlite3Fts5Dequote(char *z); /* Load the contents of the %_config table */ int sqlite3Fts5ConfigLoad(Fts5Config*, int); /* Set the value of a single config attribute */ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*); /* ** End of interface to code in fts5_config.c. **************************************************************************/ |
︙ | ︙ | |||
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 | void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); void sqlite3Fts5BufferAppendListElem(int*, Fts5Buffer*, const char*, int); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) #define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c) #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) typedef struct Fts5PoslistReader Fts5PoslistReader; struct Fts5PoslistReader { /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ int iCol; /* If (iCol>=0), this column only */ const u8 *a; /* Position list to iterate through */ int n; /* Size of buffer at a[] in bytes */ | > > > > > > | 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); void sqlite3Fts5BufferFree(Fts5Buffer*); void sqlite3Fts5BufferZero(Fts5Buffer*); void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); void sqlite3Fts5BufferAppendListElem(int*, Fts5Buffer*, const char*, int); void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) #define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c) #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) #define fts5BufferAppend32(a,b,c) sqlite3Fts5BufferAppend32(a,b,c) /* Write and decode big-endian 32-bit integer values */ void sqlite3Fts5Put32(u8*, int); int sqlite3Fts5Get32(const u8*); typedef struct Fts5PoslistReader Fts5PoslistReader; struct Fts5PoslistReader { /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ int iCol; /* If (iCol>=0), this column only */ const u8 *a; /* Position list to iterate through */ int n; /* Size of buffer at a[] in bytes */ |
︙ | ︙ | |||
294 295 296 297 298 299 300 | /* ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); | | | 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 | /* ** Called during virtual module initialization to register UDF ** fts5_decode() with SQLite */ int sqlite3Fts5IndexInit(sqlite3*); int sqlite3Fts5IndexSetCookie(Fts5Index*, int); /* ** Return the total number of entries read from the %_data table by ** this connection since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p); |
︙ | ︙ |
Changes to ext/fts5/fts5_buffer.c.
︙ | ︙ | |||
41 42 43 44 45 46 47 48 49 50 51 52 53 54 | ** Encode value iVal as an SQLite varint and append it to the buffer object ** pBuf. If an OOM error occurs, set the error code in p. */ void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return; pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal); } /* ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function ** is called, it is a no-op. */ void sqlite3Fts5BufferAppendBlob( | > > > > > > > > > > > > > > > > > > | 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | ** Encode value iVal as an SQLite varint and append it to the buffer object ** pBuf. If an OOM error occurs, set the error code in p. */ void sqlite3Fts5BufferAppendVarint(int *pRc, Fts5Buffer *pBuf, i64 iVal){ if( sqlite3Fts5BufferGrow(pRc, pBuf, 9) ) return; pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iVal); } void sqlite3Fts5Put32(u8 *aBuf, int iVal){ aBuf[0] = (iVal>>24) & 0x00FF; aBuf[1] = (iVal>>16) & 0x00FF; aBuf[2] = (iVal>> 8) & 0x00FF; aBuf[3] = (iVal>> 0) & 0x00FF; } int sqlite3Fts5Get32(const u8 *aBuf){ return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3]; } void sqlite3Fts5BufferAppend32(int *pRc, Fts5Buffer *pBuf, int iVal){ char *a; if( sqlite3Fts5BufferGrow(pRc, pBuf, 4) ) return; sqlite3Fts5Put32(&pBuf->p[pBuf->n], iVal); pBuf->n += 4; } /* ** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function ** is called, it is a no-op. */ void sqlite3Fts5BufferAppendBlob( |
︙ | ︙ |
Changes to ext/fts5/fts5_config.c.
︙ | ︙ | |||
12 13 14 15 16 17 18 19 20 21 22 23 24 25 | ** ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 1000 /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the ** input does not begin with a quote character, then this routine ** is a no-op. ** | > > > > | 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | ** ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 1000 #define FTS5_DEFAULT_AUTOMERGE 4 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the ** input does not begin with a quote character, then this routine ** is a no-op. ** |
︙ | ︙ | |||
149 150 151 152 153 154 155 156 157 158 159 160 161 162 | int rc = SQLITE_OK; /* Return code */ Fts5Config *pRet; /* New object to return */ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->db = db; pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg); pRet->zDb = fts5Strdup(azArg[1]); pRet->zName = fts5Strdup(azArg[2]); if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; | > | 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | int rc = SQLITE_OK; /* Return code */ Fts5Config *pRet; /* New object to return */ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); pRet->db = db; pRet->iCookie = -1; pRet->azCol = (char**)sqlite3_malloc(sizeof(char*) * nArg); pRet->zDb = fts5Strdup(azArg[1]); pRet->zName = fts5Strdup(azArg[2]); if( sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); rc = SQLITE_ERROR; |
︙ | ︙ | |||
303 304 305 306 307 308 309 310 311 | sqlite3_value *pVal, int *pbBadkey ){ int rc = SQLITE_OK; if( 0==sqlite3_stricmp(zKey, "cookie") ){ pConfig->iCookie = sqlite3_value_int(pVal); } else if( 0==sqlite3_stricmp(zKey, "pgsz") ){ if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ | > > | > > > | > | > > > > > > > > > > | | > > > | 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 | sqlite3_value *pVal, int *pbBadkey ){ int rc = SQLITE_OK; if( 0==sqlite3_stricmp(zKey, "cookie") ){ pConfig->iCookie = sqlite3_value_int(pVal); } else if( 0==sqlite3_stricmp(zKey, "pgsz") ){ int pgsz = 0; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ pgsz = sqlite3_value_int(pVal); } if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ if( pbBadkey ) *pbBadkey = 1; }else{ pConfig->pgsz = pgsz; } } else if( 0==sqlite3_stricmp(zKey, "automerge") ){ int nAutomerge = -1; if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ nAutomerge = sqlite3_value_int(pVal); } if( nAutomerge<0 || nAutomerge>64 ){ if( pbBadkey ) *pbBadkey = 1; }else{ if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nAutomerge = nAutomerge; } } else if( 0==sqlite3_stricmp(zKey, "rank") ){ // todo }else{ if( pbBadkey ) *pbBadkey = 1; } return rc; } /* ** Load the contents of the %_config table into memory. */ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; char *zSql; sqlite3_stmt *p = 0; int rc; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0); sqlite3_free(zSql); } assert( rc==SQLITE_OK || p==0 ); if( rc==SQLITE_OK ){ while( SQLITE_ROW==sqlite3_step(p) ){ const char *zK = (const char*)sqlite3_column_text(p, 0); sqlite3_value *pVal = sqlite3_column_value(p, 1); sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0); } rc = sqlite3_finalize(p); } if( rc==SQLITE_OK ){ pConfig->iCookie = iCookie; } return rc; } |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
38 39 40 41 42 43 44 | ** ** * extra fields in the "structure record" record the state of ongoing ** incremental merge operations. ** */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ | < | > | > > > | | 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | ** ** * extra fields in the "structure record" record the state of ongoing ** incremental merge operations. ** */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ #define FTS5_CRISIS_MERGE 16 /* Maximum number of segments to merge */ #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ /* ** Details: ** ** The %_data table managed by this module, ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** ** , contains the following 5 types of records. See the comments surrounding ** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** ** 1. Structure Records: ** ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists ** of a single 32-bit configuration cookie value followed by a list of ** SQLite varints. If the FTS table features more than one index (because ** there are one or more prefix indexes), it is guaranteed that all share ** the same cookie value. ** ** Immediately following the configuration cookie, the record begins with ** three varints: ** ** + number of levels, ** + total number of segments on all levels, ** + value of write counter. ** ** Then, for each level from 0 to nMax: ** |
︙ | ︙ | |||
284 285 286 287 288 289 290 | /* ** One object per %_data table. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ char *zDataTbl; /* Name of %_data table */ | < | 287 288 289 290 291 292 293 294 295 296 297 298 299 300 | /* ** One object per %_data table. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ char *zDataTbl; /* Name of %_data table */ int nCrisisMerge; /* Maximum allowed segments per level */ int nWorkUnit; /* Leaf pages in a "unit" of work */ /* ** Variables related to the accumulation of tokens and doclists within the ** in-memory hash tables before they are flushed to disk. */ |
︙ | ︙ | |||
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 | ** If an error occurs, *ppOut is set to NULL and an SQLite error code ** returned. Otherwise, *ppOut is set to point to the new object and ** SQLITE_OK returned. */ static int fts5StructureDecode( const u8 *pData, /* Buffer containing serialized structure */ int nData, /* Size of buffer pData in bytes */ Fts5Structure **ppOut /* OUT: Deserialized object */ ){ int rc = SQLITE_OK; int i = 0; int iLvl; int nLevel = 0; int nSegment = 0; int nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ /* Read the total number of levels and segments from the start of the ** structure record. */ | > > > > > | | 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 | ** If an error occurs, *ppOut is set to NULL and an SQLite error code ** returned. Otherwise, *ppOut is set to point to the new object and ** SQLITE_OK returned. */ static int fts5StructureDecode( const u8 *pData, /* Buffer containing serialized structure */ int nData, /* Size of buffer pData in bytes */ int *piCookie, /* Configuration cookie value */ Fts5Structure **ppOut /* OUT: Deserialized object */ ){ int rc = SQLITE_OK; int i = 0; int iLvl; int nLevel = 0; int nSegment = 0; int nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; /* Read the total number of levels and segments from the start of the ** structure record. */ i += getVarint32(&pData[i], nLevel); i += getVarint32(&pData[i], nSegment); nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel) /* aLevel[] array */ ); pRet = (Fts5Structure*)sqlite3Fts5MallocZero(&rc, nByte); |
︙ | ︙ | |||
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 | ** Fts5Index handle. If an error has already occurred when this function ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ Fts5Data *pData; /* %_data entry containing structure record */ assert( iIdx<=pConfig->nPrefix ); pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); if( !pData ) return 0; | > | > > > > | 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 | ** Fts5Index handle. If an error has already occurred when this function ** is called, it is a no-op. */ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){ Fts5Config *pConfig = p->pConfig; Fts5Structure *pRet = 0; /* Object to return */ Fts5Data *pData; /* %_data entry containing structure record */ int iCookie; /* Configuration cookie */ assert( iIdx<=pConfig->nPrefix ); pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); if( !pData ) return 0; p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); if( p->rc==SQLITE_OK && p->pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(p->pConfig, iCookie); } fts5DataRelease(pData); return pRet; } /* ** Release a reference to an Fts5Structure object returned by an earlier |
︙ | ︙ | |||
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 | ** If an error occurs, leave an error code in the Fts5Index object. If an ** error has already occurred, this function is a no-op. */ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ int nSegment; /* Total number of segments */ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ nSegment = fts5StructureCountSegments(pStruct); memset(&buf, 0, sizeof(Fts5Buffer)); fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel); fts5BufferAppendVarint(&p->rc, &buf, nSegment); fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter); for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ int iSeg; /* Used to iterate through segments */ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; | > > > > > > > | 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 | ** If an error occurs, leave an error code in the Fts5Index object. If an ** error has already occurred, this function is a no-op. */ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ int nSegment; /* Total number of segments */ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ nSegment = fts5StructureCountSegments(pStruct); memset(&buf, 0, sizeof(Fts5Buffer)); /* Append the current configuration cookie */ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; fts5BufferAppend32(&p->rc, &buf, iCookie); fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel); fts5BufferAppendVarint(&p->rc, &buf, nSegment); fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter); for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ int iSeg; /* Used to iterate through segments */ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; |
︙ | ︙ | |||
2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 | const u8 *aData, int nData ){ Fts5PageWriter *pPage = &pWriter->aWriter[0]; const u8 *a = aData; int n = nData; while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ int nReq = p->pConfig->pgsz - pPage->buf.n; int nCopy = 0; while( nCopy<nReq ){ i64 dummy; nCopy += getVarint(&a[nCopy], (u64*)&dummy); } | > | 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 | const u8 *aData, int nData ){ Fts5PageWriter *pPage = &pWriter->aWriter[0]; const u8 *a = aData; int n = nData; assert( p->pConfig->pgsz>0 ); while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ int nReq = p->pConfig->pgsz - pPage->buf.n; int nCopy = 0; while( nCopy<nReq ){ i64 dummy; nCopy += getVarint(&a[nCopy], (u64*)&dummy); } |
︙ | ︙ | |||
3175 3176 3177 3178 3179 3180 3181 | /* If nBest is still 0, then the index must be empty. */ #ifdef SQLITE_DEBUG for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ assert( pStruct->aLevel[iLvl].nSeg==0 ); } #endif | > | > > > | 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 | /* If nBest is still 0, then the index must be empty. */ #ifdef SQLITE_DEBUG for(iLvl=0; nBest==0 && iLvl<pStruct->nLevel; iLvl++){ assert( pStruct->aLevel[iLvl].nSeg==0 ); } #endif if( nBest<p->pConfig->nAutomerge && pStruct->aLevel[iBestLvl].nMerge==0 ){ break; } fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem); fts5StructurePromote(p, iBestLvl+1, pStruct); assert( nRem==0 || p->rc==SQLITE_OK ); *ppStruct = pStruct; } } |
︙ | ︙ | |||
3289 3290 3291 3292 3293 3294 3295 | pSeg->iSegid = iSegid; pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; } } | | | 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 | pSeg->iSegid = iSegid; pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; } } if( p->pConfig->nAutomerge>0 ) fts5IndexWork(p, iHash, &pStruct, pgnoLast); fts5IndexCrisisMerge(p, iHash, &pStruct); fts5StructureWrite(p, iHash, pStruct); fts5StructureRelease(pStruct); } /* ** Flush any data stored in the in-memory hash tables to the database. |
︙ | ︙ | |||
3367 3368 3369 3370 3371 3372 3373 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index)); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5Index)); p->pConfig = pConfig; | < | 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index)); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5Index)); p->pConfig = pConfig; p->nCrisisMerge = FTS5_CRISIS_MERGE; p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName); if( p->zDataTbl==0 ){ rc = SQLITE_NOMEM; }else if( bCreate ){ |
︙ | ︙ | |||
3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 | rc = p->rc; } return rc; } /* */ static void fts5DecodeStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, const u8 *pBlob, int nBlob ){ int rc; /* Return code */ Fts5Structure *p = 0; /* Decoded structure object */ | > > > > > | | 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 | rc = p->rc; } return rc; } /* ** This is part of the fts5_decode() debugging aid. ** ** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This ** function appends a human-readable representation of the same object ** to the buffer passed as the second argument. */ static void fts5DecodeStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, const u8 *pBlob, int nBlob ){ int rc; /* Return code */ Fts5Structure *p = 0; /* Decoded structure object */ rc = fts5StructureDecode(pBlob, nBlob, 0, &p); if( rc!=SQLITE_OK ){ *pRc = rc; return; } fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); |
︙ | ︙ | |||
3981 3982 3983 3984 3985 3986 3987 | int sqlite3Fts5IndexInit(sqlite3 *db){ int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); return rc; } | < < < < < < < < < < < < < | 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 | int sqlite3Fts5IndexInit(sqlite3 *db){ int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); return rc; } /* ** Iterator pMulti currently points to a valid entry (not EOF). This ** function appends a copy of the position-list of the entry pMulti ** currently points to to buffer pBuf. ** ** If an error occurs, an error code is left in p->rc. It is assumed ** no error has already occurred when this function is called. |
︙ | ︙ | |||
4403 4404 4405 4406 4407 4408 4409 4410 | /* ** Return the total number of blocks this module has read from the %_data ** table since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p){ return p->nRead; } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 | /* ** Return the total number of blocks this module has read from the %_data ** table since it was created. */ int sqlite3Fts5IndexReads(Fts5Index *p){ return p->nRead; } /* ** Set the 32-bit cookie value at the start of all structure records to ** the value passed as the second argument. ** ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; u8 aCookie[4]; int i; sqlite3Fts5Put32(aCookie, iNew); for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){ sqlite3_blob *pBlob = 0; i64 iRowid = FTS5_STRUCTURE_ROWID(i); rc = sqlite3_blob_open( pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob ); if( rc==SQLITE_OK ){ sqlite3_blob_write(pBlob, aCookie, 4, 0); rc = sqlite3_blob_close(pBlob); } } return rc; } |
Changes to ext/fts5/fts5_storage.c.
︙ | ︙ | |||
764 765 766 767 768 769 770 771 772 773 774 775 | sqlite3_stmt *pReplace = 0; int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace); if( rc==SQLITE_OK ){ sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT); sqlite3_bind_value(pReplace, 2, pVal); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } return rc; } | > > > > > > > | 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 | sqlite3_stmt *pReplace = 0; int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace); if( rc==SQLITE_OK ){ sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT); sqlite3_bind_value(pReplace, 2, pVal); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } if( rc==SQLITE_OK ){ int iNew = p->pConfig->iCookie + 1; rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); if( rc==SQLITE_OK ){ p->pConfig->iCookie = iNew; } } return rc; } |
Changes to test/fts5aa.test.
︙ | ︙ | |||
81 82 83 84 85 86 87 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); | | | 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 4.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { 1 {g f d b f} {h h e i a} 2 {f i g j e} {i j c f f} 3 {e e i f a} {e h f d f} 4 {h j f j i} {h a c f j} 5 {d b j c g} {f e i b e} |
︙ | ︙ | |||
105 106 107 108 109 110 111 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 5.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); | | | 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 5.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {i x y} { 1 {dd abc abc abc abcde} {aaa dd ddd ddd aab} 2 {dd aab d aaa b} {abcde c aaa aaa aaa} 3 {abcde dd b b dd} {abc abc d abc ddddd} 4 {aaa abcde dddd dddd abcde} {abc b b abcde abc} 5 {aab dddd d dddd c} {ddd abcde dddd abcde c} |
︙ | ︙ | |||
130 131 132 133 134 135 136 | #------------------------------------------------------------------------- # breakpoint reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); | | | | 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | #------------------------------------------------------------------------- # breakpoint reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } do_execsql_test 6.1 { INSERT INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a'); REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d'); } do_execsql_test 6.2 { INSERT INTO t1(t1) VALUES('integrity-check') } #------------------------------------------------------------------------- # reset_db expr srand(0) do_execsql_test 7.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y,z); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } proc doc {} { set v [list aaa aab abc abcde b c d dd ddd dddd ddddd] set ret [list] for {set j 0} {$j < 20} {incr j} { lappend ret [lindex $v [expr int(rand()*[llength $v])]] |
︙ | ︙ | |||
187 188 189 190 191 192 193 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); | | | | 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 8.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } do_execsql_test 8.1 { INSERT INTO t1 VALUES('the quick brown fox'); INSERT INTO t1(t1) VALUES('integrity-check'); } #------------------------------------------------------------------------- # reset_db expr srand(0) do_execsql_test 9.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3"); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } proc doc {} { set v [list aaa aab abc abcde b c d dd ddd dddd ddddd] set ret [list] for {set j 0} {$j < 20} {incr j} { lappend ret [lindex $v [expr int(rand()*[llength $v])]] |
︙ | ︙ |
Changes to test/fts5ab.test.
︙ | ︙ | |||
55 56 57 58 59 60 61 | } {} #------------------------------------------------------------------------- reset_db do_execsql_test 2.1 { CREATE VIRTUAL TABLE t1 USING fts5(x); | | | 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | } {} #------------------------------------------------------------------------- reset_db do_execsql_test 2.1 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); INSERT INTO t1 VALUES('one'); INSERT INTO t1 VALUES('two'); INSERT INTO t1 VALUES('three'); } do_catchsql_test 2.2 { SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND' |
︙ | ︙ | |||
95 96 97 98 99 100 101 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { CREATE VIRTUAL TABLE t1 USING fts5(a,b); | | | 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 | } #------------------------------------------------------------------------- # reset_db do_execsql_test 3.0 { CREATE VIRTUAL TABLE t1 USING fts5(a,b); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } foreach {tn a b} { 1 {abashed abandons abase abash abaft} {abases abased} 2 {abasing abases abaft abated abandons} {abases abandoned} 3 {abatement abash abash abated abase} {abasements abashing} 4 {abaft abasements abase abasement abasing} {abasement abases} |
︙ | ︙ |
Changes to test/fts5ac.test.
︙ | ︙ | |||
21 22 23 24 25 26 27 | ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { CREATE VIRTUAL TABLE xx USING fts5(x,y); | | | 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { CREATE VIRTUAL TABLE xx USING fts5(x,y); INSERT INTO xx(xx, rank) VALUES('pgsz', 32); } set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} 2 {l s z j k i m p s} {l w e j t j e e i t w r o p o} 3 {x g y m y m h p} {k j j b r e y y a k y} |
︙ | ︙ |
Changes to test/fts5ad.test.
︙ | ︙ | |||
51 52 53 54 55 56 57 | SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC } $res } foreach {T create} { 2 { CREATE VIRTUAL TABLE t1 USING fts5(a, b); | | | | 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 | SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC } $res } foreach {T create} { 2 { CREATE VIRTUAL TABLE t1 USING fts5(a, b); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } 3 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } } { do_test $T.1 { execsql { DROP TABLE IF EXISTS t1 } execsql $create |
︙ | ︙ |
Changes to test/fts5ae.test.
︙ | ︙ | |||
21 22 23 24 25 26 27 | ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b); | | | 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | ifcapable !fts5 { finish_test return } do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } do_execsql_test 1.1 { INSERT INTO t1 VALUES('hello', 'world'); SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC; } {1} |
︙ | ︙ |
Changes to test/fts5ah.test.
︙ | ︙ | |||
24 25 26 27 28 29 30 | #------------------------------------------------------------------------- # This file contains tests for very large doclists. # do_test 1.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) } | | | 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #------------------------------------------------------------------------- # This file contains tests for very large doclists. # do_test 1.0 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) } execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) } for {set i 1} {$i <= 10000} {incr i} { set v {x x x x x x x x x x x x x x x x x x x x} if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i} if {($i % 1577)==0} {lset v 5 W ; lappend W $i} execsql { INSERT INTO t1 VALUES($v) } } } {} |
︙ | ︙ | |||
67 68 69 70 71 72 73 | } {1} do_test 1.5 { set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }] set bwd [execsql_reads { SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC }] | | | 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | } {1} do_test 1.5 { set fwd [execsql_reads {SELECT rowid FROM t1 WHERE t1 MATCH 'x' }] set bwd [execsql_reads { SELECT rowid FROM t1 WHERE t1 MATCH 'x' ORDER BY 1 ASC }] expr {$bwd < $fwd + 12} } {1} foreach {tn q res} " 1 { SELECT rowid FROM t1 WHERE t1 MATCH 'w + x' } [list $W] 2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x + w' } [list $W] 3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W] 4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y] |
︙ | ︙ |
Changes to test/fts5aj.test.
︙ | ︙ | |||
42 43 44 45 46 47 48 | } set res } expr srand(0) do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); | | | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | } set res } expr srand(0) do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 64); } for {set iTest 0} {$iTest < 50000} {incr iTest} { if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } } set new [doc] execsql { INSERT INTO t1 VALUES($new) } if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] } |
︙ | ︙ |