Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add further tests for savepoint rollback. Fix various code issues and add missing comments in fts5_index.c. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
a805c6f7ea59a74ba3110a058ba6eb9d |
User & Date: | dan 2016-03-28 20:13:25.897 |
Context
2016-03-29
| ||
21:19 | Fix some code and test coverage issues in fts5_index.c. (Leaf check-in: 7635c68018 user: dan tags: fts5) | |
2016-03-28
| ||
20:13 | Add further tests for savepoint rollback. Fix various code issues and add missing comments in fts5_index.c. (check-in: a805c6f7ea user: dan tags: fts5) | |
15:06 | Integrate the vcreate-stmt branch into this one. (check-in: 06039d901a user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
342 343 344 345 346 347 348 | #define sqlite3Fts5IterEof(x) ((x)->bEof) /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ | < | 342 343 344 345 346 347 348 349 350 351 352 353 354 355 | #define sqlite3Fts5IterEof(x) ((x)->bEof) /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ #define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 #define FTS5INDEX_QUERY_NOOUTPUT 0x0020 |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
311 312 313 314 315 316 317 318 319 320 321 322 323 324 | /* In-memory cache of the 'structure' record */ sqlite3_stmt *pDataVersion; /* PRAGMA <db>.data_version */ i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; struct Fts5DoclistIter { u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; | > > > > > | 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 | /* In-memory cache of the 'structure' record */ sqlite3_stmt *pDataVersion; /* PRAGMA <db>.data_version */ i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ }; /* ** An iterator of this sort is used to iterate through a doclist stored ** entirely in memory. See functions fts5DoclistIterInit() and ** fts5DoclistIterNext() for details. */ struct Fts5DoclistIter { u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; |
︙ | ︙ | |||
529 530 531 532 533 534 535 536 537 538 539 540 541 542 | u8 bSkipEmpty; /* True to skip deleted entries */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. ** ** pData: ** Record containing the doclist-index data. | > > > > > > > | 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 | u8 bSkipEmpty; /* True to skip deleted entries */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; /* ** Given pointer "x" to an Fts5Iter structure, return a pointer the the ** current first component segment iterator. */ #define fts5IterSegment(x) (&(x)->aSeg [ (x)->aFirst[1].iFirst ]) /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. ** ** pData: ** Record containing the doclist-index data. |
︙ | ︙ | |||
1840 1841 1842 1843 1844 1845 1846 | /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ | | | 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 | /* ** Return true if the iterator passed as the second argument currently ** points to a delete marker. A delete marker is an entry with a 0 byte ** position-list. */ static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){ Fts5SegIter *pSeg = fts5IterSegment(pIter); return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0); } /* ** Advance iterator pIter to the next entry. ** ** This version of fts5SegIterNext() is only used by reverse iterators. |
︙ | ︙ | |||
2541 2542 2543 2544 2545 2546 2547 | ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ if( p->rc==SQLITE_OK ){ | | | 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 | ** This function is a no-op unless SQLITE_DEBUG is defined when this module ** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pFirst = fts5IterSegment(pIter); int i; assert( (pFirst->pLeaf==0)==pIter->base.bEof ); /* Check that pIter->iSwitchRowid is set correctly. */ for(i=0; i<pIter->nSeg; i++){ Fts5SegIter *p1 = &pIter->aSeg[i]; |
︙ | ︙ | |||
2814 2815 2816 2817 2818 2819 2820 | return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ static void fts5MultiIterSetEof(Fts5Iter *pIter){ | | | 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 | return 0; } /* ** Set the pIter->bEof variable based on the state of the sub-iterators. */ static void fts5MultiIterSetEof(Fts5Iter *pIter){ Fts5SegIter *pSeg = fts5IterSegment(pIter); pIter->base.bEof = pSeg->pLeaf==0; pIter->iSwitchRowid = pSeg->iRowid; } /* ** Move the iterator to the next entry. ** |
︙ | ︙ | |||
2849 2850 2851 2852 2853 2854 2855 | } if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); | | | | 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 | } if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); pSeg = fts5IterSegment(pIter); if( pSeg->pLeaf==0 ) return; } fts5AssertMultiIterSetup(p, pIter); assert( pSeg==fts5IterSegment(pIter) && pSeg->pLeaf ); if( pIter->bSkipEmpty==0 || pSeg->nPos ){ pIter->xSetOutputs(pIter, pSeg); return; } bUseFrom = 0; } } |
︙ | ︙ | |||
3423 3424 3425 3426 3427 3428 3429 | } fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ fts5MultiIterNext(p, pNew, 0, 0); }else if( pNew->base.bEof==0 ){ | < | | 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 | } fts5MultiIterSetEof(pNew); fts5AssertMultiIterSetup(p, pNew); if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ fts5MultiIterNext(p, pNew, 0, 0); }else if( pNew->base.bEof==0 ){ pNew->xSetOutputs(pNew, fts5IterSegment(pNew)); } }else{ fts5MultiIterFree(pNew); *ppOut = 0; } } |
︙ | ︙ | |||
3478 3479 3480 3481 3482 3483 3484 | } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ | | < < | | | 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 | } /* ** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ assert( p->rc || (fts5IterSegment(pIter)->pLeaf==0)==pIter->base.bEof ); return (p->rc || pIter->base.bEof); } /* ** Return the rowid of the entry that the iterator currently points ** to. If the iterator points to EOF when this function is called the ** results are undefined. */ static i64 fts5MultiIterRowid(Fts5Iter *pIter){ assert( fts5IterSegment(pIter)->pLeaf ); return fts5IterSegment(pIter)->iRowid; } /* ** Move the iterator to the next entry at or following iMatch. */ static void fts5MultiIterNextFrom( Fts5Index *p, |
︙ | ︙ | |||
3823 3824 3825 3826 3827 3828 3829 | } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; | < < < | 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 | } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5PutU16(&pPage->buf.p[2], (u16)pPage->buf.n); if( pWriter->bFirstTermInPage ){ |
︙ | ︙ | |||
4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 | const u8 *pChunk, int nChunk ){ Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); } /* ** */ static void fts5IndexMergeLevel( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ | > > > > > > > > | | 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 | const u8 *pChunk, int nChunk ){ Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; fts5WriteAppendPoslistData(p, pWriter, pChunk, nChunk); } /* ** This function reads data from level iLvl of structure (*ppStruct) and ** writes it into a segment on level (iLvl+1). If (*ppStruct) indicates ** that there is already such a merge underway, this function continues ** it. Otherwise, a new merge is started. (*ppStruct) is updated with the ** results of the merge before this function returns. ** ** When this function is called in/out parameter *pnRem contains the maximum ** number of leaf pages to write to the database. *pnRem is decremented by ** the actual number of pages written before this function returns. */ static void fts5IndexMergeLevel( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* IN/OUT: Write this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureLevel *pLvlOut; Fts5Iter *pIter = 0; /* Iterator to read input data */ int nRem = pnRem ? *pnRem : 0; /* Output leaf pages left to write */ int nInput; /* Number of input segments */ |
︙ | ︙ | |||
4222 4223 4224 4225 4226 4227 4228 | bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ | | | 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 | bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); assert( iLvl>=0 ); for(fts5MultiIterNew(p, pStruct, flags, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ Fts5SegIter *pSegIter = fts5IterSegment(pIter); int nPos; /* position-list size field value */ int nTerm; const u8 *pTerm; /* Check for key annihilation. */ if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; |
︙ | ︙ | |||
4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 | pStruct->nWriteCounter += nLeaf; nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); } } static void fts5IndexCrisismerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; int iLvl = 0; assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ fts5IndexMergeLevel(p, &pStruct, iLvl, 0); assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } *ppStruct = pStruct; } | > > > > > < < < < < < | < | > | | | | > > | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | > < > > > > > < < < < < < < < < | | | | | | < > > < | 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 | pStruct->nWriteCounter += nLeaf; nRem = (int)(p->nWorkUnit * nWork * pStruct->nLevel); fts5IndexMerge(p, ppStruct, nRem, p->pConfig->nAutomerge); } } /* ** This function is called when a new level 0 segment has just been written ** to the database. If any crisis-merge operations are required as a result, ** they are performed here. */ static void fts5IndexCrisismerge( Fts5Index *p, /* FTS5 backend object */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; int iLvl = 0; assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ fts5IndexMergeLevel(p, &pStruct, iLvl, 0); assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } *ppStruct = pStruct; } /* ** Buffer aBuf[] contains a list of varints, all small enough to fit ** in a 32-bit integer. Return the size of the largest prefix of this ** list nMax bytes or less in size. */ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ int ret; u32 dummy; ret = fts5GetVarint32(aBuf, dummy); if( ret<nMax ){ while( 1 ){ int i = fts5GetVarint32(&aBuf[ret], dummy); if( (ret + i) > nMax ) break; ret += i; } } return ret; } /* ** Flush any data stored in the in-memory hash tables to the database. ** ** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ static void fts5IndexFlush(Fts5Index *p){ if( p->nPendingData ){ Fts5Hash *pHash = p->pHash; Fts5Structure *pStruct; int iSegid; int pgnoLast = 0; /* Last leaf page number in segment */ p->nPendingData = 0; /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); pBuf = &writer.writer.buf; pPgidx = &writer.writer.pgidx; /* fts5WriteInit() should have initialized the buffers to (most likely) ** the maximum space required. */ assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); /* Begin scanning through hash table entries. This loop runs once for each ** term/doclist currently stored within the hash table. */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ /* Write the term for this entry to disk. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); assert( writer.bFirstRowidInPage==0 ); if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ i64 iRowid = 0; i64 iDelta = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOff<nDoclist ){ iOff += fts5GetVarint(&pDoclist[iOff], (u64*)&iDelta); iRowid += iDelta; if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); }else{ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); } assert( pBuf->n<=pBuf->nSpace ); if( eDetail==FTS5_DETAIL_NONE ){ if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; if( iOff<nDoclist && pDoclist[iOff]==0 ){ pBuf->p[pBuf->n++] = 0; iOff++; } } if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); } }else{ int bDummy; int nPos; int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); nCopy += nPos; if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ /* The entire poslist will fit on the current leaf. So copy ** it in one go. */ fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ /* The entire poslist will not fit on this leaf. So it needs ** to be broken into sections. The only qualification being ** that each varint must be stored contiguously. */ const u8 *pPoslist = &pDoclist[iOff]; int iPos = 0; while( p->rc==SQLITE_OK ){ int nSpace = pgsz - pBuf->n - pPgidx->n; int n = 0; if( (nCopy - iPos)<=nSpace ){ n = nCopy - iPos; }else{ n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( (pBuf->n + pPgidx->n)>=pgsz ){ fts5WriteFlushLeaf(p, &writer); } if( iPos>=nCopy ) break; } } iOff += nCopy; } } } /* TODO2: Doclist terminator written here. */ /* pBuf->p[pBuf->n++] = '\0'; */ assert( pBuf->n<=pBuf->nSpace ); sqlite3Fts5HashScanNext(pHash); } sqlite3Fts5HashClear(pHash); fts5WriteFinish(p, &writer, &pgnoLast); /* Update the Fts5Structure. It is written back to the database by the ** fts5StructureRelease() call below. */ if( pStruct->nLevel==0 ){ fts5StructureAddLevel(&p->rc, &pStruct); } fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); if( p->rc==SQLITE_OK ){ pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; pSeg->iSegid = iSegid; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; } fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); } } /* ** If argument pStruct contains fewer than two segments, NULL is returned. ** ** Otherwise, this function returns a structure reference containing all the ** same segments as argument pStruct, but arranged within levels so that ** running the merge sub-routines merges all content into a single segment. */ static Fts5Structure *fts5IndexOptimizeStruct( Fts5Index *p, /* Index object */ Fts5Structure *pStruct /* Structure to optimize */ ){ Fts5Structure *pNew = 0; int nByte = sizeof(Fts5Structure); int nSeg = pStruct->nSegment; int i; /* First figure out if this structure requires optimization. A structure ** does not require optimization if either: ** ** + it consists of fewer than two segments, or ** + all segments are on the same level, or ** + all segments except one are currently inputs to a merge operation. ** ** In the first case, return NULL. In the second and third, increment the ** ref-count on *pStruct and return a copy of the pointer to it. */ if( nSeg<2 ) return 0; for(i=0; i<pStruct->nLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ fts5StructureRef(pStruct); return pStruct; } assert( pStruct->aLevel[i].nMerge<=nThis ); } /* Allocate a new structure. Copy all segments from pStruct to level nMax+1 ** of the new structure, where nMax is the largest level in pStruct. */ nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pNew ){ Fts5StructureLevel *pLvl; nByte = nSeg * sizeof(Fts5StructureSegment); pNew->nLevel = pStruct->nLevel+1; pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; pLvl = &pNew->aLevel[pStruct->nLevel]; |
︙ | ︙ | |||
4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 | pNew = 0; } } return pNew; } int sqlite3Fts5IndexOptimize(Fts5Index *p){ Fts5Structure *pStruct; Fts5Structure *pNew = 0; assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); pStruct = fts5StructureRead(p); | > > > > > > > > > > > > | 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 | pNew = 0; } } return pNew; } /* ** Return from an Fts5Index API function that may have set the error code. */ static int fts5IndexReturn(Fts5Index *p){ int rc = p->rc; p->rc = SQLITE_OK; return rc; } /* ** The implementation of the special "VALUES('optimize')" command. */ int sqlite3Fts5IndexOptimize(Fts5Index *p){ Fts5Structure *pStruct; Fts5Structure *pNew = 0; assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); pStruct = fts5StructureRead(p); |
︙ | ︙ | |||
4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 | } } fts5StructureRelease(pStruct); } return fts5IndexReturn(p); } static void fts5AppendRowid( Fts5Index *p, i64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ UNUSED_PARAM(pUnused); fts5BufferAppendVarint(&p->rc, pBuf, iDelta); } static void fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ int nData = pMulti->base.nData; assert( nData>0 ); if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ fts5BufferSafeAppendVarint(pBuf, iDelta); fts5BufferSafeAppendVarint(pBuf, nData*2); fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); } } | > > > > > > > > > > > > > > > | > > | 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 | } } fts5StructureRelease(pStruct); } return fts5IndexReturn(p); } /* ** Append varint iDelta to buffer pBuf. ** ** This function is a no-op if Fts5Index.rc is set to other than SQLITE_OK ** when it is called. If an error occurs, Fts5Index.rc is set to an SQLite ** error code before returning. */ static void fts5AppendRowid( Fts5Index *p, i64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ UNUSED_PARAM(pUnused); fts5BufferAppendVarint(&p->rc, pBuf, iDelta); } /* ** Append varint iDelta to buffer pBuf. Then append a copy of the poslist ** currently pointed to by iterator pMulti. ** ** This function is a no-op if Fts5Index.rc is set to other than SQLITE_OK ** when it is called. If an error occurs, Fts5Index.rc is set to an SQLite ** error code before returning. */ static void fts5AppendPoslist( Fts5Index *p, i64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ int nData = pMulti->base.nData; assert( nData>0 ); if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ fts5BufferSafeAppendVarint(pBuf, iDelta); fts5BufferSafeAppendVarint(pBuf, nData*2); fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); } } /* ** Advance iterator pIter to the next rowid in its doclist. */ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; assert( pIter->aPoslist ); if( p>=pIter->aEof ){ pIter->aPoslist = 0; }else{ |
︙ | ︙ | |||
4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 | pIter->nSize = 1; } pIter->aPoslist = p; } } static void fts5DoclistIterInit( Fts5Buffer *pBuf, Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); pIter->aPoslist = pBuf->p; pIter->aEof = &pBuf->p[pBuf->n]; fts5DoclistIterNext(pIter); } | > > > > > > < < < < < < < < < < < < < < < < < < < < < < < < > > > > > > > > > > | 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 | pIter->nSize = 1; } pIter->aPoslist = p; } } /* ** Buffer pBuf contains a doclist. Set up the structure pointed to by ** pIter to iterate through it. The iterator points to the first rowid ** in the doclist (or EOF if the doclist is empty) when this function ** returns. */ static void fts5DoclistIterInit( Fts5Buffer *pBuf, Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); pIter->aPoslist = pBuf->p; pIter->aEof = &pBuf->p[pBuf->n]; fts5DoclistIterNext(pIter); } /* ** Swap the contents of buffer *p1 with that of *p2. */ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){ Fts5Buffer tmp = *p1; *p1 = *p2; *p2 = tmp; } /* ** Buffer pBuf contains a delta-encoded list of rowids (the sort stored by ** detail=none tables). ** ** When this function is called, *piOff must be set to the byte offset of a ** varint within this list, and *piRowid to the value of the previous ** rowid in the list. If there are no more rowids in the list, *piOff is ** set to -1 before returning. Otherwise, *piRowid is set to the next ** rowid in the list and *piOff to the offset of the rowid following it. */ static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ int i = *piOff; if( i>=pBuf->n ){ *piOff = -1; }else{ u64 iVal; *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal); |
︙ | ︙ | |||
4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 | fts5NextRowid(p2, &i2, &iRowid2); } } fts5BufferSwap(&out, p1); fts5BufferFree(&out); } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has | > > > > > > > > > > > > > > > > > > > | 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 | fts5NextRowid(p2, &i2, &iRowid2); } } fts5BufferSwap(&out, p1); fts5BufferFree(&out); } /* ** This macro is used by fts5MergePrefixLists(). The arguments passed should ** be of the following types: ** ** Fts5Buffer *pBuf, // Buffer to write to ** i64 iLastRowid, // IN/OUT: Previous rowid written (if any) ** i64 iRowid // Rowid to append ** ** This function appends a single rowid to the doclist stored in pBuf. The ** value of the rowid appended is iRowid. IN/OUT parameter iLastRowid ** should be set to the value of the previous rowid stored in the doclist ** when this macro is invoked. It is set to a copy of iRowid by this macro. */ #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ (iLastRowid) = (iRowid); \ } /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. ** ** If an error occurs, an error code is left in p->rc. If an error has |
︙ | ︙ | |||
4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 | fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ | > > > > > | | 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 | fts5BufferSet(&p->rc, p1, out.n, out.p); fts5BufferFree(&tmp); fts5BufferFree(&out); } } /* ** This function is used to prepare an iterator for a prefix query for ** which there is no prefix index. It assembles a doclist in memory ** and then sets up an Fts5Iter object to iterate through it. */ static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); |
︙ | ︙ | |||
5022 5023 5024 5025 5026 5027 5028 | memset(&doclist, 0, sizeof(doclist)); fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ | | | 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 | memset(&doclist, 0, sizeof(doclist)); fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ Fts5SegIter *pSeg = fts5IterSegment(p1); int nTerm = pSeg->term.n; const u8 *pTerm = pSeg->term.p; p1->xSetOutputs(p1, pSeg); assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); if( bNewTerm ){ if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break; |
︙ | ︙ | |||
5127 5128 5129 5130 5131 5132 5133 | p->pConfig->iCookie = -1; /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This | | | | 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 | p->pConfig->iCookie = -1; /* assert( p->rc==SQLITE_OK ); */ return SQLITE_OK; } /* ** The %_data table is completely empty when this function is called. This ** function populates it with the initial structure object and the initial ** version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; fts5StructureInvalidate(p); memset(&s, 0, sizeof(Fts5Structure)); fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); |
︙ | ︙ | |||
5160 5161 5162 5163 5164 5165 5166 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); if( rc==SQLITE_OK ){ p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); | | > | | | > | < | 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 | Fts5Index *p; /* New object */ *pp = p = (Fts5Index*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Index)); if( rc==SQLITE_OK ){ p->pConfig = pConfig; p->nWorkUnit = FTS5_WORK_UNIT; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( bCreate ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable(pConfig, "idx", "segid, term, pgno, PRIMARY KEY(segid, term)", 1, pzErr ); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } } } |
︙ | ︙ | |||
5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 | assert( (iCol<0)==p->bDelete ); /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ const int nChar = pConfig->aPrefix[i]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte | > > | 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 | assert( (iCol<0)==p->bDelete ); /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); /* Add an entry for each of the prefix indexes that the token is large ** enough for (e.g. for which nChar>=nPrefix). */ for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ const int nChar = pConfig->aPrefix[i]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte |
︙ | ︙ | |||
5299 5300 5301 5302 5303 5304 5305 | Fts5Colset *pColset, /* Match these columns only */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5Iter *pRet = 0; Fts5Buffer buf = {0, 0, 0}; | | > | | < < < < < < < < < > > > | < < < | 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 | Fts5Colset *pColset, /* Match these columns only */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5Iter *pRet = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. This ** flag is used by the fts5vocab module only. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ memcpy(&buf.p[1], pToken, nToken); /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to ** greater than pConfig->nPrefix to indicate that the query will be ** satisfied by scanning multiple terms in the main index. ** ** If the Fts5Config.bPrefixIndex debugging flag is set, do not use ** a prefix index even if a suitable one does exist. */ if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ if( pConfig->aPrefix[iIdx-1]==nChar ) break; } #ifdef SQLITE_DEBUG if( pConfig->bPrefixIndex==0 ) iIdx = 1+pConfig->nPrefix; #endif } if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, pColset, buf.p, nToken+1, -1, 0, &pRet ); fts5StructureRelease(pStruct); } }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); fts5IterSetOutputCb(&p->rc, pRet); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = fts5IterSegment(pRet); if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); } } if( p->rc ){ sqlite3Fts5IterClose(&pRet->base); pRet = 0; sqlite3Fts5IndexCloseReader(p); } *ppIter = &pRet->base; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); } /* ** Move to the next matching rowid. */ int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); |
︙ | ︙ | |||
5388 5389 5390 5391 5392 5393 5394 | Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(p, pIter, 0, 0); if( p->rc==SQLITE_OK ){ | | | 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 | Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *p = pIter->pIndex; assert( pIter->pIndex->rc==SQLITE_OK ); fts5MultiIterNext(p, pIter, 0, 0); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = fts5IterSegment(pIter); if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; pIter->base.bEof = 1; } } |
︙ | ︙ | |||
5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 | Fts5Iter *pIter = (Fts5Iter*)pIndexIter; fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } /* ** Return the current term. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); *pn = n-1; return &z[1]; } | > > > | 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 | Fts5Iter *pIter = (Fts5Iter*)pIndexIter; fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); return fts5IndexReturn(pIter->pIndex); } /* ** Return the current term. ** ** This function is only called as part of the fts5vocab module - not as ** part of normal full-text query processing. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); *pn = n-1; return &z[1]; } |
︙ | ︙ | |||
5460 5461 5462 5463 5464 5465 5466 | fts5DataRelease(pData); return fts5IndexReturn(p); } /* ** Replace the current "averages" record with the contents of the buffer | | > > > > > | > > > > | 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 | fts5DataRelease(pData); return fts5IndexReturn(p); } /* ** Replace the current "averages" record with the contents of the buffer ** supplied as the second argument. ** ** The averages record consists of N+1 varints, where N is the number of ** columns in the fts5 table. The first varint is the total number of ** rows in the FTS table. The second varint is the total number of tokens ** in the first column of the table, and so on. */ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ assert( p->rc==SQLITE_OK ); fts5DataWrite(p, FTS5_AVERAGES_ROWID, pData, nData); return fts5IndexReturn(p); } /* ** Return the total number of blocks this module has read from the %_data ** table (since it was created by sqlite3Fts5IndexOpen). */ int sqlite3Fts5IndexReads(Fts5Index *p){ return p->nRead; } /* ** Increment the value of the configuration cookie stored as the first ** 32-bits of the structure record in the database. This is called after ** modifying the contents of the %_config table. */ int sqlite3Fts5IndexIncrCookie(Fts5Index *p){ Fts5Structure *pStruct; pStruct = fts5StructureRead(p); p->pConfig->iCookie++; fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); return fts5IndexReturn(p); } /* ** Ensure the contents of the %_config table have been loaded into memory. */ int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ fts5StructureCache(p); return fts5IndexReturn(p); } /* ** This is called when a new read or write transaction may be being opened. ** It ensures that the in-memory cache of the structure record is valid. */ int sqlite3Fts5IndexNewTrans(Fts5Index *p){ assert( p->pStruct==0 || p->iStructVersion!=0 ); if( p->pConfig->iCookie<0 || fts5IndexDataVersion(p)!=p->iStructVersion ){ fts5StructureInvalidate(p); } return fts5IndexReturn(p); } |
︙ | ︙ | |||
5660 5661 5662 5663 5664 5665 5666 5667 | ** the index is disabled are the same. In both ASC and DESC order. ** ** This check may only be performed if the hash table is empty. This ** is because the hash table only supports a single scan query at ** a time, and the multi-iter loop from which this function is called ** is already performing such a scan. */ if( p->nPendingData==0 ){ if( iIdx>0 && rc==SQLITE_OK ){ | > > < | < | > | 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 | ** the index is disabled are the same. In both ASC and DESC order. ** ** This check may only be performed if the hash table is empty. This ** is because the hash table only supports a single scan query at ** a time, and the multi-iter loop from which this function is called ** is already performing such a scan. */ if( p->nPendingData==0 ){ int bSaved = p->pConfig->bPrefixIndex; p->pConfig->bPrefixIndex = 1; if( iIdx>0 && rc==SQLITE_OK ){ ck2 = 0; rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck2); if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; } if( iIdx>0 && rc==SQLITE_OK ){ ck2 = 0; rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck2); if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; } p->pConfig->bPrefixIndex = bSaved; } cksum3 ^= ck1; fts5BufferSet(&rc, pPrev, n, (const u8*)z); if( rc==SQLITE_OK && cksum3!=expected ){ rc = FTS5_CORRUPT; |
︙ | ︙ |
Changes to ext/fts5/test/fts5ai.test.
︙ | ︙ | |||
47 48 49 50 51 52 53 | ROLLBACK TO one; COMMIT; } do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } | | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | ROLLBACK TO one; COMMIT; } do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } #------------------------------------------------------------------------- # Test that the in-memory configuration does not become inconsistent with # respect to the on-disk configuration if a savepoint is rolled back. # proc posrowid {cmd} { $cmd xRowid } proc negrowid {cmd} { expr -1 * [$cmd xRowid] } sqlite3_fts5_create_function db posrowid posrowid sqlite3_fts5_create_function db negrowid negrowid do_execsql_test 2.1 { INSERT INTO t1(rowid, a) VALUES(1001, 'x y 1'); INSERT INTO t1(rowid, a) VALUES(1002, 'x y 2'); INSERT INTO t1(rowid, a) VALUES(1003, 'x y 3'); BEGIN; INSERT INTO t1(t1, rank) VALUES('rank', 'posrowid()'); SELECT a FROM t1('x') ORDER BY rank; } {{x y 1} {x y 2} {x y 3}} do_execsql_test 2.2 { SAVEPOINT abc; INSERT INTO t1(t1, rank) VALUES('rank', 'negrowid()'); SELECT a FROM t1('x') ORDER BY rank; } {{x y 3} {x y 2} {x y 1}} do_execsql_test 2.3 { ROLLBACK TO abc; SELECT a FROM t1('x') ORDER BY rank; COMMIT; } {{x y 1} {x y 2} {x y 3}} #------------------------------------------------------------------------- # Test that the in-memory structure does not become inconsistent with # respect to the on-disk configuration if a savepoint is rolled back. # do_execsql_test 3.1 { CREATE VIRTUAL TABLE t2 USING fts5(x, detail=%DETAIL%); INSERT INTO t2 VALUES('a 1'); INSERT INTO t2 VALUES('a 2'); INSERT INTO t2 VALUES('a 3'); SELECT count(*) FROM t2_data; } {5} do_execsql_test 3.2 { BEGIN; SAVEPOINT one; INSERT INTO t2(rowid, x) VALUES(8, 'a 8'); INSERT INTO t2(rowid, x) VALUES(7, 'a 7'); INSERT INTO t2(rowid, x) VALUES(6, 'a 6'); SELECT count(*) FROM t2_data; } {7} do_execsql_test 3.3 { INSERT INTO t2(t2) VALUES('integrity-check') } {} do_execsql_test 3.4 { ROLLBACK TO one; SELECT count(*) FROM t2_data; } {5} do_execsql_test 3.5 { INSERT INTO t2(t2) VALUES('integrity-check') } {} } ;# foreach_detail_mode finish_test |