Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Further minor optimizations to flushing fts5 data to disk. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
a07dcca9ef3821a6719ef9dbbc8ed861 |
| User & Date: | dan 2015-02-27 09:41:10 |
Context
|
2015-03-04
| ||
| 08:29 | Fix a couple of build problems. check-in: a5d5468c user: dan tags: fts5 | |
|
2015-02-27
| ||
| 09:41 | Further minor optimizations to flushing fts5 data to disk. check-in: a07dcca9 user: dan tags: fts5 | |
| 07:23 | Fix suffix and prefix compression of terms in top-level fts5 segments. And a crash that could follow an OOM condition. check-in: bb104b36 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5_index.c.
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 .... 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 .... 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 .... 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 .... 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 |
u32 dummy;
int i = fts5GetVarint32(&aBuf[ret], dummy);
if( (ret + i) > nMax ) break;
ret += i;
}
return ret;
}
/*
** Flush the contents of in-memory hash table iHash to a new level-0
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has
** already occurred, this function is a no-op.
................................................................................
/* Decide if the term fits on the current leaf. If not, flush it
** to disk. */
if( (pBuf->n + nTerm + 2) > pgsz ){
fts5WriteFlushLeaf(p, &writer);
pBuf = &writer.aWriter[0].buf;
if( (nTerm + 32) > pBuf->nSpace ){
fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
}
}
/* Write the term to the leaf. And push it up into the b-tree hierarchy */
if( writer.bFirstTermInPage==0 ){
int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm);
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre);
................................................................................
fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
pBuf = &writer.aWriter[0].buf;
assert( nPre<nTerm );
}
nSuffix = nTerm;
}
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix);
fts5BufferAppendBlob(&p->rc, pBuf,
nSuffix, (const u8*)&zTerm[nTerm-nSuffix]
);
if( pgsz>=(pBuf->n + nDoclist + 1) ){
/* The entire doclist will fit on the current leaf. */
fts5BufferAppendBlob(&p->rc, pBuf, nDoclist, pDoclist);
}else{
i64 iRowid = 0;
i64 iDelta = 0;
int iOff = 0;
int bFirstDocid = 0;
/* The entire doclist will not fit on this leaf. The following
................................................................................
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
}
assert( pBuf->n<=pBuf->nSpace );
if( (pBuf->n + nCopy) <= pgsz ){
/* The entire poslist will fit on the current leaf. So copy
** it in one go. */
fts5BufferAppendBlob(&p->rc, pBuf, nCopy, &pDoclist[iOff]);
}else{
/* The entire poslist will not fit on this leaf. So it needs
** to be broken into sections. The only qualification being
** that each varint must be stored contiguously. */
const u8 *pPoslist = &pDoclist[iOff];
int iPos = 0;
while( 1 ){
................................................................................
int nSpace = pgsz - pBuf->n;
int n;
if( (nCopy - iPos)<=nSpace ){
n = nCopy - iPos;
}else{
n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
}
fts5BufferAppendBlob(&p->rc, pBuf, n, &pPoslist[iPos]);
iPos += n;
if( iPos>=nCopy ) break;
fts5WriteFlushLeaf(p, &writer);
pBuf = &writer.aWriter[0].buf;
}
bFirstDocid = 1;
}
|
> > > > > > > < | < | | | |
3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 .... 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 .... 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 .... 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 .... 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 |
u32 dummy;
int i = fts5GetVarint32(&aBuf[ret], dummy);
if( (ret + i) > nMax ) break;
ret += i;
}
return ret;
}
#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \
memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \
pBuf->n += nBlob; \
}
/*
** Flush the contents of in-memory hash table iHash to a new level-0
** segment on disk. Also update the corresponding structure record.
**
** If an error occurs, set the Fts5Index.rc error code. If an error has
** already occurred, this function is a no-op.
................................................................................
/* Decide if the term fits on the current leaf. If not, flush it
** to disk. */
if( (pBuf->n + nTerm + 2) > pgsz ){
fts5WriteFlushLeaf(p, &writer);
pBuf = &writer.aWriter[0].buf;
if( (nTerm + 32) > pBuf->nSpace ){
fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
if( p->rc ) break;
}
}
/* Write the term to the leaf. And push it up into the b-tree hierarchy */
if( writer.bFirstTermInPage==0 ){
int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm);
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre);
................................................................................
fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
pBuf = &writer.aWriter[0].buf;
assert( nPre<nTerm );
}
nSuffix = nTerm;
}
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix);
fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix);
if( pgsz>=(pBuf->n + nDoclist + 1) ){
/* The entire doclist will fit on the current leaf. */
fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
}else{
i64 iRowid = 0;
i64 iDelta = 0;
int iOff = 0;
int bFirstDocid = 0;
/* The entire doclist will not fit on this leaf. The following
................................................................................
pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
}
assert( pBuf->n<=pBuf->nSpace );
if( (pBuf->n + nCopy) <= pgsz ){
/* The entire poslist will fit on the current leaf. So copy
** it in one go. */
fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
}else{
/* The entire poslist will not fit on this leaf. So it needs
** to be broken into sections. The only qualification being
** that each varint must be stored contiguously. */
const u8 *pPoslist = &pDoclist[iOff];
int iPos = 0;
while( 1 ){
................................................................................
int nSpace = pgsz - pBuf->n;
int n;
if( (nCopy - iPos)<=nSpace ){
n = nCopy - iPos;
}else{
n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
}
fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
iPos += n;
if( iPos>=nCopy ) break;
fts5WriteFlushLeaf(p, &writer);
pBuf = &writer.aWriter[0].buf;
}
bFirstDocid = 1;
}
|