Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Modify the way the number of leaves written and the maximum relative level are calculated in the auto-incr-merge code. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-auto-incr-merge |
Files: | files | file ages | folders |
SHA1: |
0d841c957c6ec4afecb49504177c6279 |
User & Date: | dan 2012-03-24 14:45:19.279 |
Context
2012-03-24
| ||
14:45 | Merge auto-incr-merge with incr-merge branch. (check-in: 1c68687ab6 user: dan tags: fts4-incr-merge) | |
14:45 | Modify the way the number of leaves written and the maximum relative level are calculated in the auto-incr-merge code. (Closed-Leaf check-in: 0d841c957c user: dan tags: fts4-auto-incr-merge) | |
02:20 | An attempt at automatic incremental merging for FTS4. (check-in: ed69434cd8 user: drh tags: fts4-auto-incr-merge) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
3099 3100 3101 3102 3103 3104 3105 3106 3107 | } /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); | > > | > > > > > | | | | 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 | } /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ rc = sqlite3Fts3MaxLevel(p, &mxLevel); assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * p->mxLevel; A += (A/2); if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); } sqlite3Fts3SegmentsClose(p); return rc; } /* ** Implementation of xBegin() method. This is a no-op. |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
199 200 201 202 203 204 205 | u8 bAutoincrmerge; /* True if automerge=1 */ int mxLevel; /* Maximum level seen on this transaction */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ | | | 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | u8 bAutoincrmerge; /* True if automerge=1 */ int mxLevel; /* Maximum level seen on this transaction */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ |
︙ | ︙ | |||
427 428 429 430 431 432 433 434 435 436 437 438 439 440 | int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); | > | 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 | int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
261 262 263 264 265 266 267 268 269 270 271 272 273 274 | #define SQL_MAX_LEAF_NODE_ESTIMATE 29 #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** | > | 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 | #define SQL_MAX_LEAF_NODE_ESTIMATE 29 #define SQL_DELETE_SEGDIR_ENTRY 30 #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** |
︙ | ︙ | |||
359 360 361 362 363 364 365 | /* SQL_SEGMENT_IS_APPENDABLE ** Return a single row if the segment with end_block=? is appendable. Or ** no rows otherwise. */ /* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ | | > > > > | 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 | /* SQL_SEGMENT_IS_APPENDABLE ** Return a single row if the segment with end_block=? is appendable. Or ** no rows otherwise. */ /* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ /* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt<SizeofArray(azSql) && eStmt>=0 ); |
︙ | ︙ | |||
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 | /* ** Update the Fts3Table.mxLevel field, if appropriate */ static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ iLevel %= FTS3_SEGDIR_MAXLEVEL; if( iLevel>p->mxLevel ) p->mxLevel = iLevel; } /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ | > > > > > > > > > > > > > > > > > > > > > | 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 | /* ** Update the Fts3Table.mxLevel field, if appropriate */ static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ iLevel %= FTS3_SEGDIR_MAXLEVEL; if( iLevel>p->mxLevel ) p->mxLevel = iLevel; } /* ** Find the largest relative level number in the table. If successful, set ** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, ** set *pnMax to zero and return an SQLite error code. */ int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ int rc; int mxLevel = 0; sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ mxLevel = sqlite3_column_int(pStmt, 0); } rc = sqlite3_reset(pStmt); } *pnMax = mxLevel; return rc; } /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( Fts3Table *p, /* Virtual table handle */ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ |
︙ | ︙ | |||
2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 | if( nData>0 && nData+nReq>p->nNodeSize ){ int rc; /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; /* Add the current term to the interior node tree. The term added to ** the interior tree must: ** ** a) be greater than the largest term on the leaf node just written ** to the database (still available in pWriter->zTerm), and ** | > | 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 | if( nData>0 && nData+nReq>p->nNodeSize ){ int rc; /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: ** ** a) be greater than the largest term on the leaf node just written ** to the database (still available in pWriter->zTerm), and ** |
︙ | ︙ | |||
2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 | p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } return rc; } /* ** Release all memory held by the SegmentWriter object passed as the ** first argument. */ | > | 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 | p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; } /* ** Release all memory held by the SegmentWriter object passed as the ** first argument. */ |
︙ | ︙ | |||
3007 3008 3009 3010 3011 3012 3013 | /* ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; | < | 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 | /* ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; for(i=0; rc==SQLITE_OK && i<p->nIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, |
︙ | ︙ |