Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
051c756c367837908f6691c0a36108e0 |
User & Date: | dan 2011-06-16 16:06:05.320 |
Context
2011-06-17
| ||
07:07 | Add Microsoft nmake compatible makefile; update a few test cases for Windows. (check-in: a7590af65f user: shaneh tags: trunk) | |
2011-06-16
| ||
16:06 | Fix a problem with NEAR queries executed inside a transaction that writes the FTS table. (check-in: 051c756c36 user: dan tags: trunk) | |
00:54 | Changes to #ifdefs so that the build goes correctly if the only FTS macro defined is SQLITE_ENABLE_FTS4. (check-in: a0b43a320e user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
3489 3490 3491 3492 3493 3494 3495 | ** edited in place by fts3EvalNearTrim2(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by ** fts3EvalNearTrim2(). */ while( pIter<pEnd && *pIter==0 ) pIter++; pDL->pNextDocid = pIter; | | | 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 | ** edited in place by fts3EvalNearTrim2(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by ** fts3EvalNearTrim2(). */ while( pIter<pEnd && *pIter==0 ) pIter++; pDL->pNextDocid = pIter; assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); *pbEof = 0; } } return rc; } |
︙ | ︙ | |||
4127 4128 4129 4130 4131 4132 4133 | ){ if( pExpr && *pRc==SQLITE_OK ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ | | | > | > | | | < | 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 | ){ if( pExpr && *pRc==SQLITE_OK ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ assert( pPhrase->nToken==1 ); assert( pPhrase->aToken[0].pSegcsr ); sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } pPhrase->doclist.pNextDocid = 0; pPhrase->doclist.iDocid = 0; } pExpr->iDocid = 0; pExpr->bEof = 0; pExpr->bStart = 0; fts3EvalRestart(pCsr, pExpr->pLeft, pRc); |
︙ | ︙ |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
435 436 437 438 439 440 441 442 443 444 445 446 447 448 | int nSegment; /* Size of apSegment array */ int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ /* Used by fts3.c only. */ int nCost; /* Cost of running iterator */ int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ | > | 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 | int nSegment; /* Size of apSegment array */ int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; /* Used by fts3.c only. */ int nCost; /* Cost of running iterator */ int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ |
︙ | ︙ |
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 | ); if( bEof ){ pReader->pOffsetList = 0; }else{ pReader->pOffsetList = p; } }else{ /* Pointer p currently points at the first byte of an offset list. The ** following block advances it to point one byte past the end of ** the same offset list. */ while( 1 ){ /* The following line of code (and the "p++" below the while() loop) is | > | 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 | ); if( bEof ){ pReader->pOffsetList = 0; }else{ pReader->pOffsetList = p; } }else{ char *pEnd = &pReader->aDoclist[pReader->nDoclist]; /* Pointer p currently points at the first byte of an offset list. The ** following block advances it to point one byte past the end of ** the same offset list. */ while( 1 ){ /* The following line of code (and the "p++" below the while() loop) is |
︙ | ︙ | |||
1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 | /* If required, populate the output variables with a pointer to and the ** size of the previous offset-list. */ if( ppOffsetList ){ *ppOffsetList = pReader->pOffsetList; *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } /* If there are no more entries in the doclist, set pOffsetList to ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and ** Fts3SegReader.pOffsetList to point to the next offset list before ** returning. */ | > > | | 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 | /* If required, populate the output variables with a pointer to and the ** size of the previous offset-list. */ if( ppOffsetList ){ *ppOffsetList = pReader->pOffsetList; *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } while( p<pEnd && *p==0 ) p++; /* If there are no more entries in the doclist, set pOffsetList to ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and ** Fts3SegReader.pOffsetList to point to the next offset list before ** returning. */ if( p>=pEnd ){ pReader->pOffsetList = 0; }else{ rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); if( rc==SQLITE_OK ){ sqlite3_int64 iDelta; pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); if( pTab->bDescIdx ){ |
︙ | ︙ | |||
2261 2262 2263 2264 2265 2266 2267 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } | > > > > > > > | < | < | | < | < < < | | < | < < < < < < < < < | < < < < < < < < | < < < < < < < | | 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 | p += sqlite3Fts3GetVarint32(p, &iCurrent); } *ppList = pList; *pnList = nList; } /* ** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any ** existing data). Grow the buffer if required. ** ** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered ** trying to resize the buffer, return SQLITE_NOMEM. */ static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, int nList ){ if( nList>pMsr->nBuffer ){ char *pNew; pMsr->nBuffer = nList*2; pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; } memcpy(pMsr->aBuffer, pList, nList); return SQLITE_OK; } int sqlite3Fts3MsrIncrNext( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ sqlite3_int64 *piDocid, /* OUT: Docid value */ |
︙ | ︙ | |||
2359 2360 2361 2362 2363 2364 2365 | fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); } if( nList>0 ){ | > > > | > > | > > | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < | > < | > > | | < < | < < > > > > > | > > | | < | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 | fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); } if( nList>0 ){ if( fts3SegReaderIsPending(apSegment[0]) ){ rc = fts3MsrBufferData(pMsr, pList, nList+1); if( rc!=SQLITE_OK ) return rc; *paPoslist = pMsr->aBuffer; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); }else{ *paPoslist = pList; } *piDocid = iDocid; *pnPoslist = nList; break; } } } return SQLITE_OK; } static int fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr, /* Cursor object */ const char *zTerm, /* Term searched for (or NULL) */ int nTerm /* Length of zTerm in bytes */ ){ int i; int nSeg = pCsr->nSegment; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ for(i=0; pCsr->bRestart==0 && i<pCsr->nSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; do { int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); return SQLITE_OK; } int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ pCsr->pFilter = pFilter; return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); } int sqlite3Fts3MsrIncrStart( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr, /* Cursor object */ int iCol, /* Column to match on. */ const char *zTerm, /* Term to iterate through a doclist for */ int nTerm /* Number of bytes in zTerm */ ){ int i; int rc; int nSegment = pCsr->nSegment; int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp ); assert( pCsr->pFilter==0 ); assert( zTerm && nTerm>0 ); /* Advance each segment iterator until it points to the term zTerm/nTerm. */ rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); if( rc!=SQLITE_OK ) return rc; /* Determine how many of the segments actually point to zTerm/nTerm. */ for(i=0; i<nSegment; i++){ Fts3SegReader *pSeg = pCsr->apSegment[i]; if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ break; } } pCsr->nAdvance = i; /* Advance each of the segments to point to the first docid. */ for(i=0; i<pCsr->nAdvance; i++){ rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); assert( iCol<0 || iCol<p->nColumn ); pCsr->iColFilter = iCol; return SQLITE_OK; } /* ** This function is called on a MultiSegReader that has been started using ** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also ** have been made. Calling this function puts the MultiSegReader in such ** a state that if the next two calls are: ** ** sqlite3Fts3SegReaderStart() ** sqlite3Fts3SegReaderStep() ** ** then the entire doclist for the term is available in ** MultiSegReader.aDoclist/nDoclist. */ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ int i; /* Used to iterate through segment-readers */ assert( pCsr->zTerm==0 ); assert( pCsr->nTerm==0 ); assert( pCsr->aDoclist==0 ); assert( pCsr->nDoclist==0 ); pCsr->nAdvance = 0; pCsr->bRestart = 1; for(i=0; i<pCsr->nSegment; i++){ pCsr->apSegment[i]->pOffsetList = 0; pCsr->apSegment[i]->nOffsetList = 0; pCsr->apSegment[i]->iDocid = 0; } return SQLITE_OK; } int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; |
︙ | ︙ | |||
2474 2475 2476 2477 2478 2479 2480 | } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ | > > > | > | > | | 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 | } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; } if( rc==SQLITE_OK ) rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged ** and a single term returned with the merged doclist. |
︙ | ︙ |
Changes to test/fts3auto.test.
︙ | ︙ | |||
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 | expr {[fts3_zero_long_segments t1 $limit]>0} } {1} do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } #-------------------------------------------------------------------------- # The following test cases - fts3auto-5.* - focus on using prefix indexes. # set chunkconfig [fts3_configure_incr_load 1 1] | > | < | > > | | > > | > > | 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 | expr {[fts3_zero_long_segments t1 $limit]>0} } {1} do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } #-------------------------------------------------------------------------- # The following test cases - fts3auto-5.* - focus on using prefix indexes. # set chunkconfig [fts3_configure_incr_load 1 1] foreach {tn create pending} { 2 "fts4(a, b, order=ASC, prefix=1)" 1 1 "fts4(a, b)" 1 3 "fts4(a, b, order=ASC, prefix=1,3)" 0 4 "fts4(a, b, order=DESC, prefix=2,4)" 0 5 "fts4(a, b, order=DESC, prefix=1)" 0 6 "fts4(a, b, order=ASC, prefix=1,3)" 0 } { execsql [subst { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING $create; }] if {$pending} {execsql BEGIN} foreach {a b} { "the song of songs which is solomons" "let him kiss me with the kisses of his mouth for thy love is better than wine" "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" |
︙ | ︙ | |||
563 564 565 566 567 568 569 570 571 572 573 574 | "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" "behold thou art fair my love behold thou art fair thou hast doves eyes" "behold thou art fair my beloved yea pleasant also our bed is green" "the beams of our house are cedar and our rafters of fir" } { execsql {INSERT INTO t1(a, b) VALUES($a, $b)} } do_fts3query_test 5.$tn.1.1 t1 {s*} do_fts3query_test 5.$tn.1.2 t1 {so*} do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} | > | | > > > | 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 | "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" "behold thou art fair my love behold thou art fair thou hast doves eyes" "behold thou art fair my beloved yea pleasant also our bed is green" "the beams of our house are cedar and our rafters of fir" } { execsql {INSERT INTO t1(a, b) VALUES($a, $b)} } do_fts3query_test 5.$tn.1.1 t1 {s*} do_fts3query_test 5.$tn.1.2 t1 {so*} do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} do_fts3query_test 5.$tn.1.5 t1 {a*} do_fts3query_test 5.$tn.1.6 t1 {th* NEAR/5 a* NEAR/5 w*} do_fts3query_test 5.$tn.1.7 t1 {"b* th* art* fair*"} if {$pending} {execsql COMMIT} } eval fts3_configure_incr_load $chunkconfig set sqlite_fts3_enable_parentheses $sfep finish_test |