Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts4-incr-merge |
Files: | files | file ages | folders |
SHA1: |
ab0a4f44fb67e9f0cb82297b80e728ca |
User & Date: | dan 2012-03-22 16:48:12.328 |
Context
2012-03-22
| ||
17:48 | Add test cases to fts4merge.test. (check-in: ecab208333 user: dan tags: fts4-incr-merge) | |
16:48 | Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off. (check-in: ab0a4f44fb user: dan tags: fts4-incr-merge) | |
2012-03-21
| ||
14:34 | Add fts4merge3.test, for testing that older versions of FTS4 may interoperate with incr-merge capable versions. (check-in: 903ec5126d user: dan tags: fts4-incr-merge) | |
Changes
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
62 63 64 65 66 67 68 69 70 71 72 73 74 75 | # define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize # define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold #else # define FTS3_NODE_CHUNKSIZE (4*1024) # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic ** and incremental merge operation that takes place. This is used for ** debugging FTS only, it should not usually be turned on in production ** systems. */ | > > > > > > | 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 | # define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize # define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold #else # define FTS3_NODE_CHUNKSIZE (4*1024) # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif /* ** The two values that may be meaningfully bound to the :1 parameter in ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. */ #define FTS_STAT_DOCTOTAL 0 #define FTS_STAT_INCRMERGEHINT 1 /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic ** and incremental merge operation that takes place. This is used for ** debugging FTS only, it should not usually be turned on in production ** systems. */ |
︙ | ︙ | |||
239 240 241 242 243 244 245 | #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 #define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 | | | | 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 | #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 #define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 #define SQL_REPLACE_DOCSIZE 20 #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_STAT 22 #define SQL_REPLACE_STAT 23 #define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 #define SQL_DELETE_SEGDIR_RANGE 26 #define SQL_SELECT_ALL_LANGID 27 #define SQL_FIND_MERGE_LEVEL 28 #define SQL_MAX_LEAF_NODE_ESTIMATE 29 |
︙ | ︙ | |||
301 302 303 304 305 306 307 | /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", | | | | | 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 | /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", /* 24 */ "", /* 25 */ "", /* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 27 */ "SELECT DISTINCT level / (1024 * ?) FROM %Q.'%q_segdir'", /* This statement is used to determine which level to read the input from ** when performing an incremental merge. It returns the absolute level number ** of the oldest level in the db that contains at least ? segments. Or, ** if no level in the FTS index contains more than ? segments, the statement ** returns zero rows. */ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" " ORDER BY (level %% 1024) DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See ** function fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", |
︙ | ︙ | |||
388 389 390 391 392 393 394 | *pp = pStmt; return rc; } static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ | < < < | < | < > > | > > > > > > > > > > > > | | 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 | *pp = pStmt; return rc; } static int fts3SelectDocsize( Fts3Table *pTab, /* FTS3 table handle */ sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ int rc; /* Return code */ rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iDocid); rc = sqlite3_step(pStmt); if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; pStmt = 0; }else{ rc = SQLITE_OK; } } *ppStmt = pStmt; return rc; } int sqlite3Fts3SelectDoctotal( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ sqlite3_stmt *pStmt = 0; int rc; rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; pStmt = 0; } } *ppStmt = pStmt; return rc; } int sqlite3Fts3SelectDocsize( Fts3Table *pTab, /* Fts3 table handle */ sqlite3_int64 iDocid, /* Docid to read size data for */ sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ return fts3SelectDocsize(pTab, iDocid, ppStmt); } /* ** Similar to fts3SqlStmt(). Except, after binding the parameters in ** array apVal[] to the SQL statement identified by eStmt, the statement ** is executed. ** |
︙ | ︙ | |||
3090 3091 3092 3093 3094 3095 3096 | if( *pRC ) return; a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } pBlob = (char*)&a[nStat]; | | > | 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 | if( *pRC ) return; a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } pBlob = (char*)&a[nStat]; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); if( sqlite3_step(pStmt)==SQLITE_ROW ){ fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), sqlite3_column_bytes(pStmt, 0)); }else{ memset(a, 0, sizeof(u32)*(nStat) ); } |
︙ | ︙ | |||
3119 3120 3121 3122 3123 3124 3125 | x = 0; }else{ x = x + aSzIns[i] - aSzDel[i]; } a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); | | > | | 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 | x = 0; }else{ x = x + aSzIns[i] - aSzDel[i]; } a[i+1] = x; } fts3EncodeIntArray(nStat, a, pBlob, &nBlob); rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); sqlite3_free(a); } /* ** Merge the entire database so that there is one segment for each |
︙ | ︙ | |||
3273 3274 3275 3276 3277 3278 3279 | nByte = sizeof(Fts3SegReader *) * nSeg; pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->apSegment, 0, nByte); | < > > | 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 | nByte = sizeof(Fts3SegReader *) * nSeg; pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->apSegment, 0, nByte); rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); } if( rc==SQLITE_OK ){ int i; int rc2; sqlite3_bind_int64(pStmt, 1, iAbsLevel); assert( pCsr->nSegment==0 ); for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && i<nSeg; i++){ rc = sqlite3Fts3SegReaderNew(i, 0, sqlite3_column_int64(pStmt, 1), /* segdir.start_block */ sqlite3_column_int64(pStmt, 2), /* segdir.leaves_end_block */ sqlite3_column_int64(pStmt, 3), /* segdir.end_block */ sqlite3_column_blob(pStmt, 4), /* segdir.root */ sqlite3_column_bytes(pStmt, 4), /* segdir.root */ &pCsr->apSegment[i] ); pCsr->nSegment++; } rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = rc2; } return rc; } |
︙ | ︙ | |||
3965 3966 3967 3968 3969 3970 3971 3972 | rc2 = sqlite3_reset(pSelect); if( rc==SQLITE_OK ) rc = rc2; } return rc; } | | > > > > > > > > > > | > > > > | > > > > > > > | > > | > > > | 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 | rc2 = sqlite3_reset(pSelect); if( rc==SQLITE_OK ) rc = rc2; } return rc; } /* ** Determine the largest segment index value that exists within absolute ** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus ** one before returning SQLITE_OK. Or, if there are no segments at all ** within level iAbsLevel, set *piIdx to zero. ** ** If an error occurs, return an SQLite error code. The final value of ** *piIdx is undefined in this case. */ static int fts3IncrmergeOutputIdx( Fts3Table *p, /* FTS Table handle */ sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ int *piIdx /* OUT: Next free index at iAbsLevel+1 */ ){ int rc; sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); sqlite3_step(pOutputIdx); *piIdx = sqlite3_column_int(pOutputIdx, 0); rc = sqlite3_reset(pOutputIdx); } return rc; } /* ** Allocate an appendable output segment on absolute level iAbsLevel+1 ** with idx value iIdx. ** ** In the %_segdir table, a segment is defined by the values in three ** columns: ** ** start_block ** leaves_end_block ** end_block |
︙ | ︙ | |||
3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 | ** ** In the actual code below, the value "16" is replaced with the ** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ static int fts3IncrmergeWriter( Fts3Table *p, /* Fts3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ IncrmergeWriter *pWriter /* Populate this object */ ){ int rc; /* Return Code */ int i; /* Iterator variable */ int nLeafEst; /* Blocks allocated for leaf nodes */ | > < < < < < < < < < < < < < < < < < < < | 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 | ** ** In the actual code below, the value "16" is replaced with the ** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ static int fts3IncrmergeWriter( Fts3Table *p, /* Fts3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ int iIdx, /* Index of new output segment */ Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ IncrmergeWriter *pWriter /* Populate this object */ ){ int rc; /* Return Code */ int i; /* Iterator variable */ int nLeafEst; /* Blocks allocated for leaf nodes */ sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ /* Calculate nLeafEst. */ rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ |
︙ | ︙ | |||
4186 4187 4188 4189 4190 4191 4192 | ** SQLite error code otherwise. */ static int fts3TruncateSegment( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ int iIdx, /* Index within level of segment to modify */ const char *zTerm, /* Remove terms smaller than this */ | | < | | | | | | | | | | | | | > | 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 | ** SQLite error code otherwise. */ static int fts3TruncateSegment( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ int iIdx, /* Index within level of segment to modify */ const char *zTerm, /* Remove terms smaller than this */ int nTerm /* Number of bytes in buffer zTerm */ ){ int rc = SQLITE_OK; /* Return code */ Blob root = {0,0,0}; /* New root page image */ Blob block = {0,0,0}; /* Buffer used for any other block */ sqlite3_int64 iBlock = 0; /* Block id */ sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); if( rc==SQLITE_OK ){ int rc2; /* sqlite3_reset() return code */ sqlite3_bind_int64(pFetch, 1, iAbsLevel); sqlite3_bind_int(pFetch, 2, iIdx); if( SQLITE_ROW==sqlite3_step(pFetch) ){ const char *aRoot = sqlite3_column_blob(pFetch, 4); int nRoot = sqlite3_column_bytes(pFetch, 4); iOldStart = sqlite3_column_int64(pFetch, 1); rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); } rc2 = sqlite3_reset(pFetch); if( rc==SQLITE_OK ) rc = rc2; } while( rc==SQLITE_OK && iBlock ){ char *aBlock = 0; int nBlock = 0; iNewStart = iBlock; rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); |
︙ | ︙ | |||
4269 4270 4271 4272 4273 4274 4275 | ** its data was copied to the output segment by the incrmerge operation) ** or modified in place so that it no longer contains those entries that ** have been duplicated in the output segment. */ static int fts3IncrmergeChomp( Fts3Table *p, /* FTS table handle */ sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > < < > > > > > > | | | | | | | | | > > > | | > | > | | > > > > | < | | < > > > > | | > > > > | > > > | > | | | | | | > > > > | | < < < < | > > > > > > | 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 | ** its data was copied to the output segment by the incrmerge operation) ** or modified in place so that it no longer contains those entries that ** have been duplicated in the output segment. */ static int fts3IncrmergeChomp( Fts3Table *p, /* FTS table handle */ sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ int *pnRem /* Number of segments not deleted */ ){ int i; int nRem = 0; int rc = SQLITE_OK; for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ Fts3SegReader *pSeg = 0; int j; /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding ** somewhere in the pCsr->apSegment[] array. */ for(j=0; ALWAYS(j<pCsr->nSegment); j++){ pSeg = pCsr->apSegment[j]; if( pSeg->iIdx==i ) break; } assert( j<pCsr->nSegment && pSeg->iIdx==i ); if( pSeg->aNode==0 ){ /* Seg-reader is at EOF. Remove the entire input segment. */ rc = fts3DeleteSegment(p, pSeg); if( rc==SQLITE_OK ){ rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); } *pnRem = 0; }else{ /* The incremental merge did not copy all the data from this ** segment to the upper level. The segment is modified in place ** so that it contains no keys smaller than zTerm/nTerm. */ const char *zTerm = pSeg->zTerm; int nTerm = pSeg->nTerm; rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); nRem++; } } *pnRem = nRem; return rc; } /* ** Store an incr-merge hint in the database. */ static int fts3IncrmergeHintStore( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level to read input data from */ int nMerge /* Number of segments to merge */ ){ char aBlob[FTS3_VARINT_MAX * 2]; int nBlob = 0; int rc; sqlite3_stmt *pReplace = 0; assert( p->bHasStat ); nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], iAbsLevel); nBlob += sqlite3Fts3PutVarint(&aBlob[nBlob], nMerge); rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); sqlite3_bind_blob(pReplace, 2, aBlob, nBlob, SQLITE_TRANSIENT); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } return rc; } /* ** Load an incr-merge hint from the database. ** ** The incr-merge hint, if one exists, is stored in the rowid==1 row of ** the %_stat table. */ static int fts3IncrmergeHintLoad( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 *piAbsLevel, /* Absolute level to read input data from */ int *pnMerge /* Number of segments to merge */ ){ sqlite3_stmt *pSelect = 0; int rc; *pnMerge = 0; *piAbsLevel = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); if( SQLITE_ROW==sqlite3_step(pSelect) ){ char *aHint = sqlite3_column_blob(pSelect, 0); int nHint = sqlite3_column_bytes(pSelect, 0); if( aHint ){ int i; char aBlob[FTS3_VARINT_MAX * 2]; memcpy(aBlob, aHint, MAX(sizeof(aBlob), nHint)); i = sqlite3Fts3GetVarint(aBlob, piAbsLevel); sqlite3Fts3GetVarint32(&aBlob[i], pnMerge); } } rc = sqlite3_reset(pSelect); } return rc; } /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** ** Incremental merges happen nMin segments at a time. The two ** segments to be merged are the nMin oldest segments (the ones with ** the smallest indexes) in the highest level that contains at least ** nMin segments. Multiple merges might occur in an attempt to write the ** quota of nMerge leaf blocks. */ static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ int bUseHint = 1; /* True if hint has not yet been attempted */ sqlite3_int64 iHintAbsLevel = 0;/* Hint level */ int nHintSeg = 0; /* Hint number of segments */ int nSeg = 0; /* Number of input segments */ sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ assert( nMin>=2 ); rc = fts3IncrmergeHintLoad(p, &iHintAbsLevel, &nHintSeg); if( nHintSeg==0 ) bUseHint = 0; while( rc==SQLITE_OK && nRem>0 ){ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ IncrmergeWriter *pWriter; /* Writer object */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); if( bUseHint ){ iAbsLevel = iHintAbsLevel; nSeg = nHintSeg; }else{ sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ /* Determine which level to merge segments from. Any level, from any ** prefix or language index may be selected. Stack variable iAbsLevel ** is set to the absolute level number of the level to merge from. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); sqlite3_bind_int(pFindLevel, 1, nMin); if( sqlite3_step(pFindLevel)!=SQLITE_ROW ){ /* There are no levels with nMin or more segments. Or an error has ** occurred. Either way, exit early. */ rc = sqlite3_reset(pFindLevel); iAbsLevel = 0; nSeg = 0; break; } iAbsLevel = sqlite3_column_int64(pFindLevel, 0); nSeg = nMin; sqlite3_reset(pFindLevel); } /* Allocate space for the cursor, filter and writer objects */ pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, nAlloc); pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; /* Open a cursor to iterate through the contents of the oldest nSeg ** indexes of absolute level iAbsLevel. If this cursor is opened using ** the 'hint' parameters, it is possible that there are less than nSeg ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); if( pCsr->nSegment==nSeg && SQLITE_OK==rc && SQLITE_OK ==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); if( rc==SQLITE_OK ){ if( bUseHint && iIdx>0 ){ const char *zKey = pCsr->zTerm; int nKey = pCsr->nTerm; rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); }else{ rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } } if( rc==SQLITE_OK && pWriter->nLeafEst ){ fts3LogMerge(nSeg, iAbsLevel); do { rc = fts3IncrmergeAppend(p, pWriter, pCsr); if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; }while( rc==SQLITE_ROW ); /* Update or delete the input segments */ if( rc==SQLITE_OK ){ nRem -= (1 + pWriter->nWork); rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); } } fts3IncrmergeRelease(p, pWriter, &rc); } sqlite3Fts3SegReaderFinish(pCsr); sqlite3_free(pWriter); bUseHint = 0; } /* Write the hint values into the %_stat table for the next incr-merger */ if( rc==SQLITE_OK && (iAbsLevel!=iHintAbsLevel || nHintSeg!=nSeg) ){ rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg); } return rc; } /* ** Process statements of the form: |
︙ | ︙ |
Changes to test/fts4merge.test.
︙ | ︙ | |||
59 60 61 62 63 64 65 | SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' } {123 132 213 231 312 321} } do_execsql_test 1.5 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level } { | < | < | 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 | SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three' } {123 132 213 231 312 321} } do_execsql_test 1.5 { SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level } { 2 {0 1} 3 0 } #------------------------------------------------------------------------- # Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are # handled correctly. # |
︙ | ︙ | |||
110 111 112 113 114 115 116 | } do_execsql_test 3.3 { INSERT INTO t2(t2) VALUES('merge=1000000,2'); SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level } { 0 0 | < | | | | > > > > > > > > > > | > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | } do_execsql_test 3.3 { INSERT INTO t2(t2) VALUES('merge=1000000,2'); SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level } { 0 0 2 0 3 0 4 0 6 0 } #------------------------------------------------------------------------- # Test cases 4.* # reset_db do_execsql_test 4.1 { PRAGMA page_size = 512; CREATE VIRTUAL TABLE t4 USING fts4; PRAGMA main.page_size; } {512} do_test 4.2 { foreach x {a c b d e f g h i j k l m n o p} { execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')" } execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level} } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}} foreach {tn expect} { 1 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0" 2 "0 {0 1 2 3 4 5 6 7 8 9 10 11 12} 1 0" 3 "0 {0 1 2 3 4 5 6 7 8 9 10 11} 1 0" 4 "0 {0 1 2 3 4 5 6 7 8 9 10} 1 0" 5 "0 {0 1 2 3 4 5 6 7 8 9} 1 0" 6 "0 {0 1 2 3 4 5 6 7 8} 1 0" 7 "0 {0 1 2 3 4 5 6 7} 1 0" 8 "0 {0 1 2 3 4 5 6} 1 0" 9 "0 {0 1 2 3 4 5} 1 0" } { do_execsql_test 4.3.$tn { INSERT INTO t4(t4) VALUES('merge=1,16'); SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; } $expect } do_execsql_test 4.4.1 { SELECT quote(value) FROM t4_stat WHERE rowid=1 } {X'0006'} do_execsql_test 4.4.2 { DELETE FROM t4_stat WHERE rowid=1; INSERT INTO t4(t4) VALUES('merge=1,12'); SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level; } "0 {0 1 2 3 4 5} 1 0" finish_test |