Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts4-experimental Excluding Merge-Ins
This is equivalent to a diff from 0901dcce to 3997d47b
2014-05-16
| ||
16:16 | Fixes to prevent an FTS index from growing indefinitely as the corresponding table is updated. Change the FTS 'automerge' option to allow the user to specify the number of segments that should be merged simultaneously by auto-merges. (check-in: a75f1800 user: dan tags: trunk) | |
15:48 | Add extra test to backcompat.test to ensure that old and new versions of FTS may work together on the same incremental merge operation. (Closed-Leaf check-in: 3997d47b user: dan tags: fts4-experimental) | |
14:17 | Repurpose the SQLITE_TESTCTRL_FAULT_INSTALL test-control to register a callback to be invoked by sqlite3FaultSim(). That test-control has been unused since 2008-06-20 and was never used in any official release. (check-in: 0d43a7ad user: drh tags: trunk) | |
10:30 | Fix a bug causing an incorrect segment size value to be stored if both an old and new FTS version performed work on the same incremental merge operation. (check-in: a9a2aeab user: dan tags: fts4-experimental) | |
2014-05-15
| ||
19:05 | Merge latest trunk changes with this branch. (check-in: 5809986f user: dan tags: fts4-experimental) | |
2014-05-12
| ||
22:36 | Improve a comment in the MSVC makefile. No changes to code. (check-in: 0901dcce user: mistachkin tags: trunk) | |
21:12 | Add VSIX packaging support for Windows Phone 8.1 using Visual Studio 2013 Update 2. (check-in: 013738f3 user: mistachkin tags: trunk) | |
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
1329 1330 1331 1332 1333 1334 1335 | p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; | | | 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 | p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); |
︙ | ︙ | |||
3298 3299 3300 3301 3302 3303 3304 | ** segments. */ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); | > | > > | | 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 | ** segments. */ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK && p->nLeafAdd>(nMinMerge/16) && p->nAutoincrmerge && p->nAutoincrmerge!=0xff ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ rc = sqlite3Fts3MaxLevel(p, &mxLevel); assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; } /* ** If it is currently unknown whether or not the FTS table has an %_stat |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
206 207 208 209 210 211 212 | const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ | | | | 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ u8 bHasStat; /* True if %_stat table exists (2==unknown) */ |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
189 190 191 192 193 194 195 196 197 198 199 200 201 202 | char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except ** the leaf nodes). These functions and type are only ever used by code ** within the fts3SegWriterXXX() family of functions described above. | > | 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 | char *zTerm; /* Pointer to previous term buffer */ int nTerm; /* Number of bytes in zTerm */ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ i64 nLeafData; /* Number of bytes of leaf data written */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except ** the leaf nodes). These functions and type are only ever used by code ** within the fts3SegWriterXXX() family of functions described above. |
︙ | ︙ | |||
263 264 265 266 267 268 269 270 271 272 273 274 275 276 | #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** | > > > > | 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 | #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 #define SQL_SELECT_LEVEL_RANGE2 37 #define SQL_UPDATE_LEVEL_IDX 38 #define SQL_UPDATE_LEVEL 39 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** |
︙ | ︙ | |||
365 366 367 368 369 370 371 | /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ | | > > > > > > > > > > > | 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 | /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ /* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", /* Return segments in order from oldest to newest.*/ /* 37 */ "SELECT level, idx, end_block " "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " "ORDER BY level DESC, idx ASC", /* Update statements used while promoting segments */ /* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " "WHERE level=? AND idx=?", /* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
︙ | ︙ | |||
1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 | static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); | > > | > > > > > | 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 | static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); if( nLeafData==0 ){ sqlite3_bind_int64(pStmt, 5, iEndBlock); }else{ char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); if( !zEnd ) return SQLITE_NOMEM; sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } return rc; } |
︙ | ︙ | |||
2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 | nSuffix = nTerm; nReq = 1 + /* varint containing prefix size */ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ char *aNew = sqlite3_realloc(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; | > > > | 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 | nSuffix = nTerm; nReq = 1 + /* varint containing prefix size */ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } /* Increase the total number of bytes written to account for the new entry. */ pWriter->nLeafData += nReq; /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ char *aNew = sqlite3_realloc(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; |
︙ | ︙ | |||
2313 2314 2315 2316 2317 2318 2319 | iLastLeaf = pWriter->iFree; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ | | | | | | 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 | iLastLeaf = pWriter->iFree; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir(p, iLevel, iIdx, 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; } /* ** Release all memory held by the SegmentWriter object passed as the |
︙ | ︙ | |||
2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 | getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents ** of the %_segdir table. */ static int fts3DeleteSegment( | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 | getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } /* ** iAbsLevel is an absolute level that may be assumed to exist within ** the database. This function checks if it is the largest level number ** within its index. Assuming no error occurs, *pbMax is set to 1 if ** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK ** is returned. If an error occurs, an error code is returned and the ** final value of *pbMax is undefined. */ static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ /* Set pStmt to the compiled version of: ** ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); sqlite3_bind_int64(pStmt, 2, ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL ); *pbMax = 0; if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; } return sqlite3_reset(pStmt); } /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents ** of the %_segdir table. */ static int fts3DeleteSegment( |
︙ | ︙ | |||
2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 | sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 | sqlite3_free(pCsr->aBuffer); pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } /* ** Decode the "end_block" field, selected by column iCol of the SELECT ** statement passed as the first argument. ** ** The "end_block" field may contain either an integer, or a text field ** containing the text representation of two non-negative integers separated ** by one or more space (0x20) characters. In the first case, set *piEndBlock ** to the integer value and *pnByte to zero before returning. In the second, ** set *piEndBlock to the first value and *pnByte to the second. */ static void fts3ReadEndBlockField( sqlite3_stmt *pStmt, int iCol, i64 *piEndBlock, i64 *pnByte ){ const unsigned char *zText = sqlite3_column_text(pStmt, iCol); if( zText ){ int i; int iMul = 1; i64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } *piEndBlock = iVal; while( zText[i]==' ' ) i++; iVal = 0; if( zText[i]=='-' ){ i++; iMul = -1; } for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } *pnByte = (iVal * (i64)iMul); } } /* ** A segment of size nByte bytes has just been written to absolute level ** iAbsLevel. Promote any segments that should be promoted as a result. */ static int fts3PromoteSegments( Fts3Table *p, /* FTS table handle */ int iAbsLevel, /* Absolute level just updated */ sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ ){ int rc = SQLITE_OK; sqlite3_stmt *pRange; rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); if( rc==SQLITE_OK ){ int bOk = 0; int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; i64 nLimit = (nByte*3)/2; /* Loop through all entries in the %_segdir table corresponding to ** segments in this index on levels greater than iAbsLevel. If there is ** at least one such segment, and it is possible to determine that all ** such segments are smaller than nLimit bytes in size, they will be ** promoted to level iAbsLevel. */ sqlite3_bind_int(pRange, 1, iAbsLevel+1); sqlite3_bind_int(pRange, 2, iLast); while( SQLITE_ROW==sqlite3_step(pRange) ){ i64 nSize, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); if( nSize<=0 || nSize>nLimit ){ /* If nSize==0, then the %_segdir.end_block field does not not ** contain a size value. This happens if it was written by an ** old version of FTS. In this case it is not possible to determine ** the size of the segment, and so segment promotion does not ** take place. */ bOk = 0; break; } bOk = 1; } rc = sqlite3_reset(pRange); if( bOk ){ int iIdx = 0; sqlite3_stmt *pUpdate1; sqlite3_stmt *pUpdate2; if( rc==SQLITE_OK ){ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); } if( rc==SQLITE_OK ){ rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); } if( rc==SQLITE_OK ){ /* Loop through all %_segdir entries for segments in this index with ** levels equal to or greater than iAbsLevel. As each entry is visited, ** updated it to set (level = -1) and (idx = N), where N is 0 for the ** oldest segment in the range, 1 for the next oldest, and so on. ** ** In other words, move all segments being promoted to level -1, ** setting the "idx" fields as appropriate to keep them in the same ** order. The contents of level -1 (which is never used, except ** transiently here), will be moved back to level iAbsLevel below. */ sqlite3_bind_int(pRange, 1, iAbsLevel); while( SQLITE_ROW==sqlite3_step(pRange) ){ sqlite3_bind_int(pUpdate1, 1, iIdx++); sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); sqlite3_step(pUpdate1); rc = sqlite3_reset(pUpdate1); if( rc!=SQLITE_OK ){ sqlite3_reset(pRange); break; } } } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pRange); } /* Move level -1 to level iAbsLevel */ if( rc==SQLITE_OK ){ sqlite3_bind_int(pUpdate2, 1, iAbsLevel); sqlite3_step(pUpdate2); rc = sqlite3_reset(pUpdate2); } } } return rc; } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level ** currently present in the database. ** |
︙ | ︙ | |||
2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 | int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( iIndex>=0 && iIndex<p->nIndex ); rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } | > > > > > > | < < < > > | > | > | > > > > > > | | > | > > > > | 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 | int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( iLevel<FTS3_SEGDIR_MAXLEVEL ); assert( iIndex>=0 && iIndex<p->nIndex ); rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); if( rc!=SQLITE_OK ) goto finished; } if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } iNewLevel = iMaxLevel; bIgnoreEmpty = 1; }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevel<getAbsoluteLevel(p, iLangid, iIndex,FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ rc = sqlite3Fts3SegReaderStep(p, &csr); if( rc!=SQLITE_ROW ) break; rc = fts3SegWriterAdd(p, &pWriter, 1, csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); } if( rc!=SQLITE_OK ) goto finished; assert( pWriter || bIgnoreEmpty ); if( iLevel!=FTS3_SEGCURSOR_PENDING ){ rc = fts3DeleteSegdir( p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment ); if( rc!=SQLITE_OK ) goto finished; } if( pWriter ){ rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); if( rc==SQLITE_OK ){ if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevel<iMaxLevel ){ rc = fts3PromoteSegments(p, iNewLevel, pWriter->nLeafData); } } } finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); if( rc==SQLITE_ROW ){ p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; }else if( rc==SQLITE_DONE ){ p->nAutoincrmerge = 0; } rc = sqlite3_reset(pStmt); } } return rc; } /* |
︙ | ︙ | |||
3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 | struct IncrmergeWriter { int nLeafEst; /* Space allocated for leaf blocks */ int nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* ** An object of the following type is used to read data from a single ** FTS segment node. See the following functions: ** | > > | 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 | struct IncrmergeWriter { int nLeafEst; /* Space allocated for leaf blocks */ int nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* ** An object of the following type is used to read data from a single ** FTS segment node. See the following functions: ** |
︙ | ︙ | |||
3760 3761 3762 3763 3764 3765 3766 3767 | nSuffix = nTerm; nSpace = 1; nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); | > < | 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 | nSuffix = nTerm; nSpace = 1; nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; pLeaf->block.a[0] = '\0'; } rc = fts3AppendToNode( &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist |
︙ | ︙ | |||
3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 | if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } sqlite3_free(pRoot->block.a); sqlite3_free(pRoot->key.a); *pRc = rc; | > | 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 | if( rc==SQLITE_OK ){ rc = fts3WriteSegdir(p, pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } sqlite3_free(pRoot->block.a); sqlite3_free(pRoot->key.a); *pRc = rc; |
︙ | ︙ | |||
3961 3962 3963 3964 3965 3966 3967 | /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); | | > > > > | 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 | /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); if( pWriter->nLeafData<0 ){ pWriter->nLeafData = pWriter->nLeafData * -1; } pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ return sqlite3_reset(pSelect); } /* Check for the zero-length marker in the %_segments table */ |
︙ | ︙ | |||
4562 4563 4564 4565 4566 4567 4568 | return SQLITE_OK; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** | | | | | | | 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 | return SQLITE_OK; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** ** Incremental merges happen nMin segments at a time. The segments ** to be merged are the nMin oldest segments (the ones with the smallest ** values for the _segdir.idx field) in the highest level that contains ** at least nMin segments. Multiple merges might occur in an attempt to ** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ IncrmergeWriter *pWriter; /* Writer object */ |
︙ | ︙ | |||
4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 | pCsr = (Fts3MultiSegReader *)&pFilter[1]; rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ | > | 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 | pCsr = (Fts3MultiSegReader *)&pFilter[1]; rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ |
︙ | ︙ | |||
4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 | ** indexes of absolute level iAbsLevel. If this cursor is opened using ** the 'hint' parameters, it is possible that there are less than nSeg ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } if( SQLITE_OK==rc && pCsr->nSegment==nSeg && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ | > > > > > > > > > > > > > < < < | | | | | | < | 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 | ** indexes of absolute level iAbsLevel. If this cursor is opened using ** the 'hint' parameters, it is possible that there are less than nSeg ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; if( rc==SQLITE_OK ){ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); assert( bUseHint==1 || bUseHint==0 ); if( iIdx==0 || (bUseHint && iIdx==1) ){ int bIgnore; rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); if( bIgnore ){ pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; } } } if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } if( SQLITE_OK==rc && pCsr->nSegment==nSeg && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ if( bUseHint && iIdx>0 ){ const char *zKey = pCsr->zTerm; int nKey = pCsr->nTerm; rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); }else{ rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ fts3LogMerge(nSeg, iAbsLevel); do { rc = fts3IncrmergeAppend(p, pWriter, pCsr); if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); |
︙ | ︙ | |||
4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 | if( nSeg!=0 ){ bDirtyHint = 1; fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); } /* Write the hint values into the %_stat table for the next incr-merger */ if( bDirtyHint && rc==SQLITE_OK ){ | > > > > > > | 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 | if( nSeg!=0 ){ bDirtyHint = 1; fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } if( nSeg!=0 ){ pWriter->nLeafData = pWriter->nLeafData * -1; } fts3IncrmergeRelease(p, pWriter, &rc); if( nSeg==0 && pWriter->bNoLeafData==0 ){ fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); } } sqlite3Fts3SegReaderFinish(pCsr); } /* Write the hint values into the %_stat table for the next incr-merger */ if( bDirtyHint && rc==SQLITE_OK ){ |
︙ | ︙ | |||
4769 4770 4771 4772 4773 4774 4775 | */ static int fts3DoAutoincrmerge( Fts3Table *p, /* FTS3 table handle */ const char *zParam /* Nul-terminated string containing boolean */ ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; | | > > > | | 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 | */ static int fts3DoAutoincrmerge( Fts3Table *p, /* FTS3 table handle */ const char *zParam /* Nul-terminated string containing boolean */ ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; p->nAutoincrmerge = fts3Getint(&zParam); if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ p->nAutoincrmerge = 8; } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); if( rc ) return rc; } rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; } /* ** Return a 64-bit checksum for the FTS index entry specified by the |
︙ | ︙ |
Changes to test/backcompat.test.
︙ | ︙ | |||
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | } proc sql1 sql { code1 [list db eval $sql] } proc sql2 sql { code2 [list db eval $sql] } code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } } array set ::incompatible [list] proc do_allbackcompat_test {script} { foreach bin $::BC(binaries) { set nErr [set_test_counter errors] | > > > > > > > > > > > > | 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 | } proc sql1 sql { code1 [list db eval $sql] } proc sql2 sql { code2 [list db eval $sql] } code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } foreach c {code1 code2} { $c { set v [split [db version] .] if {[llength $v]==3} {lappend v 0} set ::sqlite_libversion [format \ "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3] ] } } uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } } array set ::incompatible [list] proc do_allbackcompat_test {script} { foreach bin $::BC(binaries) { set nErr [set_test_counter errors] |
︙ | ︙ | |||
377 378 379 380 381 382 383 384 385 386 387 388 389 390 | 6 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'aa'" 7 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH '44'" 8 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'a*'" } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } } } } #------------------------------------------------------------------------- # Test that Rtree tables may be read/written by different versions of # SQLite. | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 | 6 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'aa'" 7 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH '44'" 8 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'a*'" } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } # Now test that an incremental merge can be started by one version # and finished by another. And that the integrity-check still # passes. do_test backcompat-3.8 { sql1 { DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE t2 USING fts3(words); } code1 [list source $testdir/genesis.tcl] code1 { fts_kjv_genesis } sql1 { INSERT INTO t2 SELECT words FROM t1; INSERT INTO t2 SELECT words FROM t1; INSERT INTO t2 SELECT words FROM t1; INSERT INTO t2 SELECT words FROM t1; INSERT INTO t2 SELECT words FROM t1; INSERT INTO t2 SELECT words FROM t1; SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; } } {0 {0 1 2 3 4 5}} if {[code1 { set ::sqlite_libversion }] >=3071200 && [code2 { set ::sqlite_libversion }] >=3071200 } { do_test backcompat-3.9 { sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); } sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); } sql2 { SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; } } {0 {0 1} 1 0} do_test backcompat-3.10 { sql1 { INSERT INTO t2(t2) VALUES('integrity-check') } sql2 { INSERT INTO t2(t2) VALUES('integrity-check') } } {} } } } } #------------------------------------------------------------------------- # Test that Rtree tables may be read/written by different versions of # SQLite. |
︙ | ︙ |
Changes to test/fts3d.test.
︙ | ︙ | |||
209 210 211 212 213 214 215 | SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \ {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] | > | | | | | | | | | | 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 | SELECT OFFSETS(t1) FROM t1 WHERE t1 MATCH 'this OR that OR was OR a OR is OR test' ORDER BY docid; } } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \ {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] puts [db eval {SELECT c FROM t1 } ] check_terms_all fts3d-4.1 {a four is test that this was} check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts3d-4.1.2 four {} check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]} #check_doclist_all fts3d-4.1.4 one {} check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts3d-4.1.6 that {[2 0[0]]} check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]} #check_doclist_all fts3d-4.1.8 three {} #check_doclist_all fts3d-4.1.9 two {} check_doclist_all fts3d-4.1.10 was {[2 0[1]]} check_terms fts3d-4.2 0 0 {a four test that was} check_doclist fts3d-4.2.1 0 0 a {[2 0[2]]} check_doclist fts3d-4.2.2 0 0 four {[2]} check_doclist fts3d-4.2.3 0 0 test {[2 0[3]]} check_doclist fts3d-4.2.4 0 0 that {[2 0[0]]} check_doclist fts3d-4.2.5 0 0 was {[2 0[1]]} check_terms fts3d-4.3 0 1 {a four is test this} check_doclist fts3d-4.3.1 0 1 a {[3 0[2]]} check_doclist fts3d-4.3.2 0 1 four {[3]} check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]} check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]} check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]} check_terms fts3d-4.4 1 0 {a four is test that this was} check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]} check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]} #check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]} check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} #check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} #check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. do_test fts3d-4.5 { execsql { SELECT OPTIMIZE(t1) FROM t1 LIMIT 1; |
︙ | ︙ |
Added test/fts4growth.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 | # 2014 May 12 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS4 module. # # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts4growth # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { finish_test return } source $testdir/genesis.tcl do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; } do_test 1.2 { foreach L { {"See here, young man," said Mulga Bill, "from Walgett to the sea,} {From Conroy's Gap to Castlereagh, there's none can ride like me.} {I'm good all round at everything as everybody knows,} {Although I'm not the one to talk -- I hate a man that blows.} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { SELECT end_block, length(root) FROM x1_segdir } } {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115} do_execsql_test 1.3 { INSERT INTO x1(x1) VALUES('optimize'); SELECT level, end_block, length(root) FROM x1_segdir; } {0 {0 394} 394} do_test 1.4 { foreach L { {But riding is my special gift, my chiefest, sole delight;} {Just ask a wild duck can it swim, a wildcat can it fight.} {There's nothing clothed in hair or hide, or built of flesh or steel,} {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,} {But what I'll sit, while hide will hold and girths and straps are tight:} {I'll ride this here two-wheeled concern right straight away at sight."} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { INSERT INTO x1(x1) VALUES('merge=4,4'); SELECT level, end_block, length(root) FROM x1_segdir; } } {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2} do_execsql_test 1.5 { SELECT length(block) FROM x1_segments; } {658 {}} do_test 1.6 { foreach L { {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,} {That perched above Dead Man's Creek, beside the mountain road.} {He turned the cycle down the hill and mounted for the fray,} {But 'ere he'd gone a dozen yards it bolted clean away.} {It left the track, and through the trees, just like a silver steak,} {It whistled down the awful slope towards the Dead Man's Creek.} {It shaved a stump by half an inch, it dodged a big white-box:} {The very wallaroos in fright went scrambling up the rocks,} {The wombats hiding in their caves dug deeper underground,} {As Mulga Bill, as white as chalk, sat tight to every bound.} {It struck a stone and gave a spring that cleared a fallen tree,} {It raced beside a precipice as close as close could be;} {And then as Mulga Bill let out one last despairing shriek} {It made a leap of twenty feet into the Dead Man's Creek.} } { execsql { INSERT INTO x1 VALUES($L) } } execsql { SELECT level, end_block, length(root) FROM x1_segdir; } } {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117} do_execsql_test 1.7 { SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130); } {1377} #------------------------------------------------------------------------- # do_execsql_test 2.1 { CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE x2 USING fts4; } fts_kjv_genesis do_test 2.2 { foreach id [db eval {SELECT docid FROM t1}] { execsql { INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id } } foreach id [db eval {SELECT docid FROM t1}] { execsql { INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id } if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break } } {} do_execsql_test 2.3 { SELECT count(*) FROM x2_segdir WHERE level=2; SELECT count(*) FROM x2_segdir WHERE level=3; } {6 0} do_execsql_test 2.4 { INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT count(*) FROM x2_segdir WHERE level=2; SELECT count(*) FROM x2_segdir WHERE level=3; } {6 1} do_execsql_test 2.5 { SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; INSERT INTO x2(x2) VALUES('merge=4,4'); SELECT end_block FROM x2_segdir WHERE level=3; } {{3828 -3430} {3828 -10191} {3828 -14109}} do_execsql_test 2.6 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE blockid BETWEEN start_block AND leaves_end_block AND level=3 } {14109} do_execsql_test 2.7 { INSERT INTO x2(x2) VALUES('merge=1000,4'); SELECT end_block FROM x2_segdir WHERE level=3; } {{3828 86120}} do_execsql_test 2.8 { SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE blockid BETWEEN start_block AND leaves_end_block AND level=3 } {86120} #-------------------------------------------------------------------------- # Test that delete markers are removed from FTS segments when possible. # It is only possible to remove delete markers when the output of the # merge operation will become the oldest segment in the index. # # 3.1 - when the oldest segment is created by an 'optimize'. # 3.2 - when the oldest segment is created by an incremental merge. # 3.3 - by a crisis merge. # proc insert_doc {args} { foreach iDoc $args { set L [lindex { {In your eagerness to engage the Trojans,} {don’t any of you charge ahead of others,} {trusting in your strength and horsemanship.} {And don’t lag behind. That will hurt our charge.} {Any man whose chariot confronts an enemy’s} {should thrust with his spear at him from there.} {That’s the most effective tactic, the way} {men wiped out city strongholds long ago —} {their chests full of that style and spirit.} } [expr $iDoc%9]] execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) } } } proc delete_doc {args} { foreach iDoc $args { execsql { DELETE FROM x3 WHERE docid = $iDoc } } } proc second {x} { lindex $x 1 } db func second second do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 } do_test 3.1.1 { db transaction { insert_doc 1 2 3 4 5 6 } execsql { SELECT level, idx, second(end_block) FROM x3_segdir } } {0 0 412} do_test 3.1.2 { delete_doc 1 2 3 4 5 6 execsql { SELECT count(*) FROM x3_segdir } } {0} do_test 3.1.3 { db transaction { insert_doc 1 2 3 4 5 6 7 8 9 delete_doc 9 8 7 } execsql { SELECT level, idx, second(end_block) FROM x3_segdir } } {0 0 591 0 1 65 0 2 72 0 3 76} do_test 3.1.4 { execsql { INSERT INTO x3(x3) VALUES('optimize') } execsql { SELECT level, idx, second(end_block) FROM x3_segdir } } {0 0 412} do_test 3.2.1 { execsql { DELETE FROM x3 } insert_doc 8 7 6 5 4 3 2 1 delete_doc 7 8 execsql { SELECT count(*) FROM x3_segdir } } {10} do_test 3.2.2 { execsql { INSERT INTO x3(x3) VALUES('merge=500,10') } execsql { SELECT level, idx, second(end_block) FROM x3_segdir } } {1 0 412} # This assumes the crisis merge happens when there are already 16 # segments and one more is added. # do_test 3.3.1 { execsql { DELETE FROM x3 } insert_doc 1 2 3 4 5 6 7 8 9 10 11 delete_doc 11 10 9 8 7 execsql { SELECT count(*) FROM x3_segdir } } {16} do_test 3.3.2 { insert_doc 12 execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 } } {1 0 412} #-------------------------------------------------------------------------- # Check a theory on a bug in fts4 - that segments with idx==0 were not # being incrementally merged correctly. Theory turned out to be false. # do_execsql_test 4.1 { DROP TABLE IF EXISTS x4; DROP TABLE IF EXISTS t1; CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE x4 USING fts4(words); } do_test 4.2 { fts_kjv_genesis execsql { INSERT INTO x4 SELECT words FROM t1 } execsql { INSERT INTO x4 SELECT words FROM t1 } } {} do_execsql_test 4.3 { SELECT level, idx, second(end_block) FROM x4_segdir } {0 0 117483 0 1 118006} do_execsql_test 4.4 { INSERT INTO x4(x4) VALUES('merge=10,2'); SELECT count(*) FROM x4_segdir; } {3} do_execsql_test 4.5 { INSERT INTO x4(x4) VALUES('merge=10,2'); SELECT count(*) FROM x4_segdir; } {3} do_execsql_test 4.6 { INSERT INTO x4(x4) VALUES('merge=1000,2'); SELECT count(*) FROM x4_segdir; } {1} #-------------------------------------------------------------------------- # Check that segments are not promoted if the "end_block" field does not # contain a size. # do_execsql_test 5.1 { DROP TABLE IF EXISTS x2; DROP TABLE IF EXISTS t1; CREATE TABLE t1(docid, words); CREATE VIRTUAL TABLE x2 USING fts4; } fts_kjv_genesis proc first {L} {lindex $L 0} db func first first do_test 5.2 { foreach r [db eval { SELECT rowid FROM t1 }] { execsql { INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r } } foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] { execsql { DELETE FROM x2 WHERE docid = $d } } execsql { INSERT INTO x2(x2) VALUES('optimize'); SELECT level, idx, end_block FROM x2_segdir } } {2 0 {752 1926}} do_execsql_test 5.3 { UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER ); SELECT end_block, typeof(end_block) FROM x2_segdir; } {752 integer} do_execsql_test 5.4 { INSERT INTO x2 SELECT words FROM t1 LIMIT 50; SELECT level, idx, end_block FROM x2_segdir } {2 0 752 0 0 {758 5174}} do_execsql_test 5.5 { UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2; INSERT INTO x2 SELECT words FROM t1 LIMIT 40; SELECT level, idx, end_block FROM x2_segdir } {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}} proc t1_to_x2 {} { foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { execsql { DELETE FROM x2 WHERE docid=$id; INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id; } } } #-------------------------------------------------------------------------- # Check that segments created by auto-merge are not promoted until they # are completed. # do_execsql_test 6.1 { CREATE VIRTUAL TABLE x5 USING fts4; INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0; INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25; INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50; INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75; SELECT count(*) FROM x5_segdir } {4} do_execsql_test 6.2 { INSERT INTO x5(x5) VALUES('merge=2,4'); SELECT level, idx, end_block FROM x5_segdir; } {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}} do_execsql_test 6.3 { INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100; SELECT level, idx, end_block FROM x5_segdir; } { 0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297} } do_execsql_test 6.4 { INSERT INTO x5(x5) VALUES('merge=200,4'); SELECT level, idx, end_block FROM x5_segdir; } {0 0 {1329 8297} 1 0 {1320 28009}} do_execsql_test 6.5 { INSERT INTO x5 SELECT words FROM t1; SELECT level, idx, end_block FROM x5_segdir; } { 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} } #-------------------------------------------------------------------------- # Ensure that if part of an incremental merge is performed by an old # version that does not support storing segment sizes in the end_block # field, no size is stored in the final segment (as it would be incorrect). # do_execsql_test 7.1 { CREATE VIRTUAL TABLE x6 USING fts4; INSERT INTO x6 SELECT words FROM t1; INSERT INTO x6 SELECT words FROM t1; INSERT INTO x6 SELECT words FROM t1; INSERT INTO x6 SELECT words FROM t1; INSERT INTO x6 SELECT words FROM t1; INSERT INTO x6 SELECT words FROM t1; SELECT level, idx, end_block FROM x6_segdir; } { 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} } do_execsql_test 7.2 { INSERT INTO x6(x6) VALUES('merge=25,4'); SELECT level, idx, end_block FROM x6_segdir; } { 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} 1 0 {16014 -51226} } do_execsql_test 7.3 { UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1; SELECT level, idx, end_block FROM x6_segdir; } { 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} 1 0 16014 } do_execsql_test 7.4 { INSERT INTO x6(x6) VALUES('merge=25,4'); SELECT level, idx, end_block FROM x6_segdir; } { 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} 1 0 16014 } do_execsql_test 7.5 { INSERT INTO x6(x6) VALUES('merge=2500,4'); SELECT level, idx, end_block FROM x6_segdir; } { 0 0 {598 118006} 0 1 {718 118006} 1 0 16014 } do_execsql_test 7.6 { INSERT INTO x6(x6) VALUES('merge=2500,2'); SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir; } { 2 0 23695 24147 {41262 633507} } do_execsql_test 7.7 { SELECT sum(length(block)) FROM x6_segments WHERE blockid BETWEEN 23695 AND 24147 } {633507} finish_test |
Added test/fts4growth2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | # 2014 May 12 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS4 module. # # set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts4growth2 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts3 { finish_test return } source $testdir/genesis.tcl do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } fts_kjv_genesis proc tt {val} { execsql { DELETE FROM x1 WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); INSERT INTO x1(docid, content) SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; } #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] } do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts4; INSERT INTO x1(x1) VALUES('automerge=2'); } do_test 1.2 { for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; SELECT count(*) FROM x1_segdir WHERE level=3; } } {3 1} do_test 1.3 { for {set i 0} {$i < 40} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir; SELECT count(*) FROM x1_segdir WHERE level=2; } } {2 1} #------------------------------------------------------------------------- # do_execsql_test 2.1 { DELETE FROM t1 WHERE rowid>16; DROP TABLE IF EXISTS x1; CREATE VIRTUAL TABLE x1 USING fts4; } db func second second proc second {L} {lindex $L 1} for {set tn 0} {$tn < 40} {incr tn} { do_test 2.2.$tn { for {set i 0} {$i < 100} {incr i} { tt 0 ; tt 1 ; tt 2 ; tt 3 } execsql { SELECT max(level) FROM x1_segdir } } {1} } finish_test |
Changes to test/fts4merge4.test.
︙ | ︙ | |||
49 50 51 52 53 54 55 56 57 58 | execsql {INSERT INTO t1 VALUES('a b c d e f g h i j k l');} } } {} do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 sqlite3_enable_shared_cache $::enable_shared_cache finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | execsql {INSERT INTO t1 VALUES('a b c d e f g h i j k l');} } } {} do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 #------------------------------------------------------------------------- # Now test that the automerge=? option appears to work. # do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; } set doc "" foreach c1 "a b c d e f g h i j" { foreach c2 "a b c d e f g h i j" { foreach c3 "a b c d e f g h i j" { lappend doc "$c1$c2$c3" } } } set doc [string repeat $doc 10] foreach {tn am expected} { 1 {automerge=2} {1 1 2 1 4 1 6 1} 2 {automerge=4} {1 2 2 1 3 1} 3 {automerge=8} {0 4 1 3 2 1} 4 {automerge=1} {0 4 1 3 2 1} } { foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} { do_test 2.2.$tn.$tn2 { execsql { DELETE FROM t2 } execsql { INSERT INTO t2(t2) VALUES($am) }; eval $openclose for {set i 0} {$i < 100} {incr i} { execsql { BEGIN; INSERT INTO t2 VALUES($doc); INSERT INTO t2 VALUES($doc); INSERT INTO t2 VALUES($doc); INSERT INTO t2 VALUES($doc); INSERT INTO t2 VALUES($doc); COMMIT; } } execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } } [list {*}$expected] } } sqlite3_enable_shared_cache $::enable_shared_cache finish_test |
Changes to test/permutations.test.
︙ | ︙ | |||
108 109 110 111 112 113 114 115 116 117 118 119 120 121 | savepoint4.test savepoint6.test select9.test speed1.test speed1p.test speed2.test speed3.test speed4.test speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test thread003.test thread004.test thread005.test trans2.test vacuum3.test incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] } ############################################################################# # Start of tests | > | 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 | savepoint4.test savepoint6.test select9.test speed1.test speed1p.test speed2.test speed3.test speed4.test speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test thread003.test thread004.test thread005.test trans2.test vacuum3.test incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test fts4growth.test fts4growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] } ############################################################################# # Start of tests |
︙ | ︙ | |||
192 193 194 195 196 197 198 199 200 201 202 203 204 205 | fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test } test_suite "nofaultsim" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ | > | 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 | fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test fts4growth.test fts4growth2.test } test_suite "nofaultsim" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ |
︙ | ︙ |
Changes to test/trace2.test.
︙ | ︙ | |||
132 133 134 135 136 137 138 139 140 141 142 143 144 145 | "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" "-- SELECT DISTINCT level / (1024 * ?) FROM 'main'.'x1_segdir'" | > | 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC" } do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" "-- SELECT DISTINCT level / (1024 * ?) FROM 'main'.'x1_segdir'" |
︙ | ︙ |