/ Check-in [83dc1ff7]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further optimizations for fts5 prefix queries without a prefix index.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 83dc1ff7fa010715ca7f406a572f4ee444a967d7
User & Date: dan 2015-10-07 19:06:21
Context
2015-10-08
02:44
Remove two unused lines of code - discovered by scan-build. check-in: 77b707b7 user: drh tags: trunk
2015-10-07
19:06
Further optimizations for fts5 prefix queries without a prefix index. check-in: 83dc1ff7 user: dan tags: trunk
17:06
Fix harmless compiler warning in FTS5. check-in: 13adcd03 user: mistachkin tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_index.c.

   309    309   struct Fts5DoclistIter {
   310    310     u8 *aEof;                       /* Pointer to 1 byte past end of doclist */
   311    311   
   312    312     /* Output variables. aPoslist==0 at EOF */
   313    313     i64 iRowid;
   314    314     u8 *aPoslist;
   315    315     int nPoslist;
          316  +  int nSize;
   316    317   };
   317    318   
   318    319   /*
   319    320   ** The contents of the "structure" record for each index are represented
   320    321   ** using an Fts5Structure record in memory. Which uses instances of the 
   321    322   ** other Fts5StructureXXX types as components.
   322    323   */
................................................................................
  4044   4045   ** If an error occurs, an error code is left in p->rc. It is assumed
  4045   4046   ** no error has already occurred when this function is called.
  4046   4047   */
  4047   4048   static int fts5MultiIterPoslist(
  4048   4049     Fts5Index *p,
  4049   4050     Fts5IndexIter *pMulti,
  4050   4051     Fts5Colset *pColset,
  4051         -  int bSz,                        /* Append a size field before the data */
  4052   4052     Fts5Buffer *pBuf
  4053   4053   ){
  4054   4054     if( p->rc==SQLITE_OK ){
  4055   4055       int iSz;
  4056   4056       int iData;
  4057   4057   
  4058   4058       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4059   4059       assert( fts5MultiIterEof(p, pMulti)==0 );
  4060   4060   
  4061         -    if( bSz ){
  4062         -      /* WRITEPOSLISTSIZE */
  4063         -      iSz = pBuf->n;
  4064         -      fts5BufferAppendVarint(&p->rc, pBuf, pSeg->nPos*2);
  4065         -      iData = pBuf->n;
  4066         -    }
         4061  +    /* WRITEPOSLISTSIZE */
         4062  +    iSz = pBuf->n;
         4063  +    fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2);
         4064  +    iData = pBuf->n;
  4067   4065   
  4068   4066       fts5SegiterPoslist(p, pSeg, pColset, pBuf);
  4069   4067   
  4070         -    if( bSz && pColset ){
         4068  +    if( pColset ){
  4071   4069         int nActual = pBuf->n - iData;
  4072   4070         if( nActual!=pSeg->nPos ){
  4073   4071           /* WRITEPOSLISTSIZE */
  4074   4072           if( nActual==0 ){
  4075   4073             return 1;
  4076   4074           }else{
  4077   4075             int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2));
................................................................................
  4082   4080       }
  4083   4081     }
  4084   4082   
  4085   4083     return 0;
  4086   4084   }
  4087   4085   
  4088   4086   static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
  4089         -  u8 *p = pIter->aPoslist + pIter->nPoslist;
         4087  +  u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist;
  4090   4088   
  4091   4089     assert( pIter->aPoslist );
  4092   4090     if( p>=pIter->aEof ){
  4093   4091       pIter->aPoslist = 0;
  4094   4092     }else{
  4095   4093       i64 iDelta;
  4096   4094   
  4097   4095       p += fts5GetVarint(p, (u64*)&iDelta);
  4098   4096       pIter->iRowid += iDelta;
  4099   4097   
  4100   4098       /* Read position list size */
  4101   4099       if( p[0] & 0x80 ){
  4102   4100         int nPos;
  4103         -      p += fts5GetVarint32(p, nPos);
         4101  +      pIter->nSize = fts5GetVarint32(p, nPos);
  4104   4102         pIter->nPoslist = (nPos>>1);
  4105   4103       }else{
  4106   4104         pIter->nPoslist = ((int)(p[0])) >> 1;
  4107         -      p++;
         4105  +      pIter->nSize = 1;
  4108   4106       }
  4109   4107   
  4110   4108       pIter->aPoslist = p;
  4111   4109     }
  4112   4110   }
  4113   4111   
  4114   4112   static void fts5DoclistIterInit(
................................................................................
  4170   4168       sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n);
  4171   4169       fts5DoclistIterInit(p1, &i1);
  4172   4170       fts5DoclistIterInit(p2, &i2);
  4173   4171       while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){
  4174   4172         if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid<i2.iRowid) ){
  4175   4173           /* Copy entry from i1 */
  4176   4174           fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4177         -        /* WRITEPOSLISTSIZE */
  4178         -        fts5BufferSafeAppendVarint(&out, i1.nPoslist * 2);
  4179         -        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist);
         4175  +        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
  4180   4176           fts5DoclistIterNext(&i1);
  4181   4177         }
  4182   4178         else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
  4183   4179           /* Copy entry from i2 */
  4184   4180           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4185         -        /* WRITEPOSLISTSIZE */
  4186         -        fts5BufferSafeAppendVarint(&out, i2.nPoslist * 2);
  4187         -        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist);
         4181  +        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
  4188   4182           fts5DoclistIterNext(&i2);
  4189   4183         }
  4190   4184         else{
  4191   4185           i64 iPos1 = 0;
  4192   4186           i64 iPos2 = 0;
  4193   4187           int iOff1 = 0;
  4194   4188           int iOff2 = 0;
         4189  +        u8 *a1 = &i1.aPoslist[i1.nSize];
         4190  +        u8 *a2 = &i2.aPoslist[i2.nSize];
  4195   4191   
  4196   4192           Fts5PoslistWriter writer;
  4197   4193           memset(&writer, 0, sizeof(writer));
  4198   4194   
  4199   4195           /* Merge the two position lists. */ 
  4200   4196           fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4201   4197           fts5BufferZero(&tmp);
  4202   4198   
  4203         -        sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
  4204         -        sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
         4199  +        sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4200  +        sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4205   4201   
  4206   4202           while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){
  4207   4203             i64 iNew;
  4208   4204             if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
  4209   4205               iNew = iPos1;
  4210         -            sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1, &iPos1);
         4206  +            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4211   4207             }else{
  4212   4208               iNew = iPos2;
  4213         -            sqlite3Fts5PoslistNext64(i2.aPoslist, i2.nPoslist, &iOff2, &iPos2);
         4209  +            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4214   4210               if( iPos1==iPos2 ){
  4215         -              sqlite3Fts5PoslistNext64(i1.aPoslist, i1.nPoslist, &iOff1,&iPos1);
         4211  +              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
  4216   4212               }
  4217   4213             }
  4218   4214             p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
  4219   4215           }
  4220   4216   
  4221   4217           /* WRITEPOSLISTSIZE */
  4222   4218           fts5BufferSafeAppendVarint(&out, tmp.n * 2);
................................................................................
  4286   4282           iLastRowid = 0;
  4287   4283         }
  4288   4284   
  4289   4285         if( 0==sqlite3Fts5BufferGrow(&p->rc, &doclist, 9) ){
  4290   4286           int iSave = doclist.n;
  4291   4287           assert( doclist.n!=0 || iLastRowid==0 );
  4292   4288           fts5BufferSafeAppendVarint(&doclist, iRowid - iLastRowid);
  4293         -        if( fts5MultiIterPoslist(p, p1, pColset, 1, &doclist) ){
         4289  +        if( fts5MultiIterPoslist(p, p1, pColset, &doclist) ){
  4294   4290             doclist.n = iSave;
  4295   4291           }else{
  4296   4292             iLastRowid = iRowid;
  4297   4293           }
  4298   4294         }
  4299   4295       }
  4300   4296   
................................................................................
  5235   5231       i64 iRowid = fts5MultiIterRowid(pIter);
  5236   5232       char *z = (char*)fts5MultiIterTerm(pIter, &n);
  5237   5233   
  5238   5234       /* If this is a new term, query for it. Update cksum3 with the results. */
  5239   5235       fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
  5240   5236   
  5241   5237       poslist.n = 0;
  5242         -    fts5MultiIterPoslist(p, pIter, 0, 0, &poslist);
         5238  +    fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist);
  5243   5239       while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
  5244   5240         int iCol = FTS5_POS2COLUMN(iPos);
  5245   5241         int iTokOff = FTS5_POS2OFFSET(iPos);
  5246   5242         cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
  5247   5243       }
  5248   5244     }
  5249   5245     fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);