Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Further minor optimizations to flushing fts5 data to disk. |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
a07dcca9ef3821a6719ef9dbbc8ed861 |
User & Date: | dan 2015-02-27 09:41:10 |
Context
2015-03-04
| ||
08:29 | Fix a couple of build problems. check-in: a5d5468c user: dan tags: fts5 | |
2015-02-27
| ||
09:41 | Further minor optimizations to flushing fts5 data to disk. check-in: a07dcca9 user: dan tags: fts5 | |
07:23 | Fix suffix and prefix compression of terms in top-level fts5 segments. And a crash that could follow an OOM condition. check-in: bb104b36 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5_index.c.
3396 3396 u32 dummy; 3397 3397 int i = fts5GetVarint32(&aBuf[ret], dummy); 3398 3398 if( (ret + i) > nMax ) break; 3399 3399 ret += i; 3400 3400 } 3401 3401 return ret; 3402 3402 } 3403 + 3404 +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ 3405 + assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \ 3406 + memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \ 3407 + pBuf->n += nBlob; \ 3408 +} 3403 3409 3404 3410 /* 3405 3411 ** Flush the contents of in-memory hash table iHash to a new level-0 3406 3412 ** segment on disk. Also update the corresponding structure record. 3407 3413 ** 3408 3414 ** If an error occurs, set the Fts5Index.rc error code. If an error has 3409 3415 ** already occurred, this function is a no-op. ................................................................................ 3456 3462 /* Decide if the term fits on the current leaf. If not, flush it 3457 3463 ** to disk. */ 3458 3464 if( (pBuf->n + nTerm + 2) > pgsz ){ 3459 3465 fts5WriteFlushLeaf(p, &writer); 3460 3466 pBuf = &writer.aWriter[0].buf; 3461 3467 if( (nTerm + 32) > pBuf->nSpace ){ 3462 3468 fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); 3469 + if( p->rc ) break; 3463 3470 } 3464 3471 } 3465 3472 3466 3473 /* Write the term to the leaf. And push it up into the b-tree hierarchy */ 3467 3474 if( writer.bFirstTermInPage==0 ){ 3468 3475 int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm); 3469 3476 pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre); ................................................................................ 3476 3483 fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); 3477 3484 pBuf = &writer.aWriter[0].buf; 3478 3485 assert( nPre<nTerm ); 3479 3486 } 3480 3487 nSuffix = nTerm; 3481 3488 } 3482 3489 pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix); 3483 - fts5BufferAppendBlob(&p->rc, pBuf, 3484 - nSuffix, (const u8*)&zTerm[nTerm-nSuffix] 3485 - ); 3490 + fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix); 3486 3491 3487 3492 if( pgsz>=(pBuf->n + nDoclist + 1) ){ 3488 3493 /* The entire doclist will fit on the current leaf. */ 3489 - fts5BufferAppendBlob(&p->rc, pBuf, nDoclist, pDoclist); 3494 + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); 3490 3495 }else{ 3491 3496 i64 iRowid = 0; 3492 3497 i64 iDelta = 0; 3493 3498 int iOff = 0; 3494 3499 int bFirstDocid = 0; 3495 3500 3496 3501 /* The entire doclist will not fit on this leaf. The following ................................................................................ 3512 3517 pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta); 3513 3518 } 3514 3519 assert( pBuf->n<=pBuf->nSpace ); 3515 3520 3516 3521 if( (pBuf->n + nCopy) <= pgsz ){ 3517 3522 /* The entire poslist will fit on the current leaf. So copy 3518 3523 ** it in one go. */ 3519 - fts5BufferAppendBlob(&p->rc, pBuf, nCopy, &pDoclist[iOff]); 3524 + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); 3520 3525 }else{ 3521 3526 /* The entire poslist will not fit on this leaf. So it needs 3522 3527 ** to be broken into sections. The only qualification being 3523 3528 ** that each varint must be stored contiguously. */ 3524 3529 const u8 *pPoslist = &pDoclist[iOff]; 3525 3530 int iPos = 0; 3526 3531 while( 1 ){ ................................................................................ 3527 3532 int nSpace = pgsz - pBuf->n; 3528 3533 int n; 3529 3534 if( (nCopy - iPos)<=nSpace ){ 3530 3535 n = nCopy - iPos; 3531 3536 }else{ 3532 3537 n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); 3533 3538 } 3534 - fts5BufferAppendBlob(&p->rc, pBuf, n, &pPoslist[iPos]); 3539 + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); 3535 3540 iPos += n; 3536 3541 if( iPos>=nCopy ) break; 3537 3542 fts5WriteFlushLeaf(p, &writer); 3538 3543 pBuf = &writer.aWriter[0].buf; 3539 3544 } 3540 3545 bFirstDocid = 1; 3541 3546 }