/ Check-in [a07dcca9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further minor optimizations to flushing fts5 data to disk.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: a07dcca9ef3821a6719ef9dbbc8ed861fa005035
User & Date: dan 2015-02-27 09:41:10
Context
2015-03-04
08:29
Fix a couple of build problems. check-in: a5d5468c user: dan tags: fts5
2015-02-27
09:41
Further minor optimizations to flushing fts5 data to disk. check-in: a07dcca9 user: dan tags: fts5
07:23
Fix suffix and prefix compression of terms in top-level fts5 segments. And a crash that could follow an OOM condition. check-in: bb104b36 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

  3396   3396       u32 dummy;
  3397   3397       int i = fts5GetVarint32(&aBuf[ret], dummy);
  3398   3398       if( (ret + i) > nMax ) break;
  3399   3399       ret += i;
  3400   3400     }
  3401   3401     return ret;
  3402   3402   }
         3403  +
         3404  +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \
         3405  +  assert( pBuf->nSpace>=(pBuf->n+nBlob) );             \
         3406  +  memcpy(&pBuf->p[pBuf->n], pBlob, nBlob);             \
         3407  +  pBuf->n += nBlob;                                    \
         3408  +}
  3403   3409   
  3404   3410   /*
  3405   3411   ** Flush the contents of in-memory hash table iHash to a new level-0 
  3406   3412   ** segment on disk. Also update the corresponding structure record.
  3407   3413   **
  3408   3414   ** If an error occurs, set the Fts5Index.rc error code. If an error has 
  3409   3415   ** already occurred, this function is a no-op.
................................................................................
  3456   3462         /* Decide if the term fits on the current leaf. If not, flush it
  3457   3463         ** to disk.  */
  3458   3464         if( (pBuf->n + nTerm + 2) > pgsz ){
  3459   3465           fts5WriteFlushLeaf(p, &writer);
  3460   3466           pBuf = &writer.aWriter[0].buf;
  3461   3467           if( (nTerm + 32) > pBuf->nSpace ){
  3462   3468             fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
         3469  +          if( p->rc ) break;
  3463   3470           }
  3464   3471         }
  3465   3472   
  3466   3473         /* Write the term to the leaf. And push it up into the b-tree hierarchy */
  3467   3474         if( writer.bFirstTermInPage==0 ){
  3468   3475           int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, zTerm);
  3469   3476           pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nPre);
................................................................................
  3476   3483             fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm);
  3477   3484             pBuf = &writer.aWriter[0].buf;
  3478   3485             assert( nPre<nTerm );
  3479   3486           }
  3480   3487           nSuffix = nTerm;
  3481   3488         }
  3482   3489         pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], nSuffix);
  3483         -      fts5BufferAppendBlob(&p->rc, pBuf, 
  3484         -          nSuffix, (const u8*)&zTerm[nTerm-nSuffix]
  3485         -      );
         3490  +      fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix);
  3486   3491   
  3487   3492         if( pgsz>=(pBuf->n + nDoclist + 1) ){
  3488   3493           /* The entire doclist will fit on the current leaf. */
  3489         -        fts5BufferAppendBlob(&p->rc, pBuf, nDoclist, pDoclist);
         3494  +        fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
  3490   3495         }else{
  3491   3496           i64 iRowid = 0;
  3492   3497           i64 iDelta = 0;
  3493   3498           int iOff = 0;
  3494   3499           int bFirstDocid = 0;
  3495   3500   
  3496   3501           /* The entire doclist will not fit on this leaf. The following 
................................................................................
  3512   3517               pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
  3513   3518             }
  3514   3519             assert( pBuf->n<=pBuf->nSpace );
  3515   3520   
  3516   3521             if( (pBuf->n + nCopy) <= pgsz ){
  3517   3522               /* The entire poslist will fit on the current leaf. So copy
  3518   3523               ** it in one go. */
  3519         -            fts5BufferAppendBlob(&p->rc, pBuf, nCopy, &pDoclist[iOff]);
         3524  +            fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy);
  3520   3525             }else{
  3521   3526               /* The entire poslist will not fit on this leaf. So it needs
  3522   3527               ** to be broken into sections. The only qualification being
  3523   3528               ** that each varint must be stored contiguously.  */
  3524   3529               const u8 *pPoslist = &pDoclist[iOff];
  3525   3530               int iPos = 0;
  3526   3531               while( 1 ){
................................................................................
  3527   3532                 int nSpace = pgsz - pBuf->n;
  3528   3533                 int n;
  3529   3534                 if( (nCopy - iPos)<=nSpace ){
  3530   3535                   n = nCopy - iPos;
  3531   3536                 }else{
  3532   3537                   n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
  3533   3538                 }
  3534         -              fts5BufferAppendBlob(&p->rc, pBuf, n, &pPoslist[iPos]);
         3539  +              fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
  3535   3540                 iPos += n;
  3536   3541                 if( iPos>=nCopy ) break;
  3537   3542                 fts5WriteFlushLeaf(p, &writer);
  3538   3543                 pBuf = &writer.aWriter[0].buf;
  3539   3544               }
  3540   3545               bFirstDocid = 1;
  3541   3546             }