Index: ext/fts5/fts5_buffer.c ================================================================== --- ext/fts5/fts5_buffer.c +++ ext/fts5/fts5_buffer.c @@ -231,17 +231,19 @@ Fts5PoslistWriter *pWriter, i64 iPos ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; int rc = SQLITE_OK; - if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ - fts5BufferAppendVarint(&rc, pBuf, 1); - fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32)); - pWriter->iPrev = (iPos & colmask); + if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){ + if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + pWriter->iPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2); + pWriter->iPrev = iPos; } - fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2); - pWriter->iPrev = iPos; return rc; } void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -305,13 +305,11 @@ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { - u8 *a; - int n; - int i; + u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; @@ -3704,14 +3702,19 @@ } } return ret; } -#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ - assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \ - memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \ - pBuf->n += nBlob; \ +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ } /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -3987,55 +3990,68 @@ fts5SegiterPoslist(p, pSeg, pBuf); } } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ - if( pIter->in ){ - int bDummy; - if( pIter->i ){ - i64 iDelta; - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta); - pIter->iRowid += iDelta; - }else{ - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid); - } - pIter->i += fts5GetPoslistSize( - &pIter->a[pIter->i], &pIter->nPoslist, &bDummy - ); - pIter->aPoslist = &pIter->a[pIter->i]; - pIter->i += pIter->nPoslist; - }else{ - pIter->aPoslist = 0; + u8 *p = pIter->aPoslist + pIter->nPoslist; + + assert( pIter->aPoslist ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + p += fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); + }else{ + pIter->nPoslist = ((int)(p[0])) >> 1; + p++; + } + + pIter->aPoslist = p; } } static void fts5DoclistIterInit( Fts5Buffer *pBuf, Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); - pIter->a = pBuf->p; - pIter->n = pBuf->n; + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; fts5DoclistIterNext(pIter); } +#if 0 /* ** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. */ static void fts5MergeAppendDocid( - int *pRc, /* IN/OUT: Error code */ Fts5Buffer *pBuf, /* Buffer to write to */ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ i64 iRowid /* Rowid to append */ ){ - if( pBuf->n==0 ){ - fts5BufferAppendVarint(pRc, pBuf, iRowid); - }else{ - fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid); - } + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); *piLastRowid = iRowid; } +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. @@ -4055,57 +4071,64 @@ Fts5Buffer out; Fts5Buffer tmp; memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); + sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n); fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowidrc, &out, &iLastRowid, i1.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist); + fts5BufferSafeAppendVarint(&out, i1.nPoslist * 2); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist); fts5DoclistIterNext(&i1); } else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist); + fts5BufferSafeAppendVarint(&out, i2.nPoslist * 2); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist); fts5DoclistIterNext(&i2); } else{ - Fts5PoslistReader r1; - Fts5PoslistReader r2; + i64 iPos1 = 0; + i64 iPos2 = 0; + int iOff1 = 0; + int iOff2 = 0; + Fts5PoslistWriter writer; - memset(&writer, 0, sizeof(writer)); /* Merge the two position lists. */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); - sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1); - sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2); - while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){ + + sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1); + sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2); + + while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){ i64 iNew; - if( r2.bEof || (r1.bEof==0 && r1.iPos=0 && iPos1rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); } /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2); - fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p); + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); } } @@ -4163,18 +4186,23 @@ }else{ fts5MergePrefixLists(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } + iLastRowid = 0; } - fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid); - fts5MultiIterPoslist(p, p1, 1, &doclist); + if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){ + fts5MergeAppendDocid(&doclist, iLastRowid, iRowid); + fts5MultiIterPoslist(p, p1, 1, &doclist); + } } for(i=0; irc==SQLITE_OK ){ + fts5MergePrefixLists(p, &doclist, &aBuf[i]); + } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);