/ Check-in [22589018]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add tests for and remove unreachable branches from fts5 in order to restore test coverage.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 22589018ac3321f7bd89ce9fa69e57eae395e324
User & Date: dan 2016-02-05 21:09:26
Context
2016-02-06
14:14
Fix an assert() to have a test instead of a side effect check-in: a2952231 user: pdr tags: trunk
2016-02-05
21:09
Add tests for and remove unreachable branches from fts5 in order to restore test coverage. check-in: 22589018 user: dan tags: trunk
19:18
Further streamlining of fts5 prefix query code. check-in: ca91bd8a user: dan tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_buffer.c.

   232    232   }
   233    233   
   234    234   int sqlite3Fts5PoslistWriterAppend(
   235    235     Fts5Buffer *pBuf, 
   236    236     Fts5PoslistWriter *pWriter,
   237    237     i64 iPos
   238    238   ){
   239         -  static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
   240    239     int rc;
   241    240     if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
   242    241     sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
   243    242     return SQLITE_OK;
   244    243   }
   245    244   
   246    245   void *sqlite3Fts5MallocZero(int *pRc, int nByte){

Changes to ext/fts5/fts5_index.c.

   514    514   
   515    515     /* Invoked to set output variables. */
   516    516     void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
   517    517   
   518    518     int nSeg;                       /* Size of aSeg[] array */
   519    519     int bRev;                       /* True to iterate in reverse order */
   520    520     u8 bSkipEmpty;                  /* True to skip deleted entries */
   521         -  u8 bFiltered;                   /* True if column-filter already applied */
   522    521   
   523    522     i64 iSwitchRowid;               /* Firstest rowid of other than aFirst[1] */
   524    523     Fts5CResult *aFirst;            /* Current merge state (see above) */
   525    524     Fts5SegIter aSeg[1];            /* Array of segment iterators */
   526    525   };
   527    526   
   528    527   
................................................................................
  2026   2025       int iPoslist;
  2027   2026       if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
  2028   2027         iPoslist = pIter->iTermLeafOffset;
  2029   2028       }else{
  2030   2029         iPoslist = 4;
  2031   2030       }
  2032   2031       fts5IndexSkipVarint(pLeaf->p, iPoslist);
  2033         -    assert( p->pConfig->eDetail==FTS5_DETAIL_NONE || iPoslist==(
  2034         -        pIter->iLeafOffset - sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel)
  2035         -    ));
  2036   2032       pIter->iLeafOffset = iPoslist;
  2037   2033   
  2038   2034       /* If this condition is true then the largest rowid for the current
  2039   2035       ** term may not be stored on the current page. So search forward to
  2040   2036       ** see where said rowid really is.  */
  2041   2037       if( pIter->iEndofDoclist>=pLeaf->szLeaf ){
  2042   2038         int pgno;
................................................................................
  3091   3087   ** column filters are specified.
  3092   3088   */
  3093   3089   static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
  3094   3090     pIter->base.iRowid = pSeg->iRowid;
  3095   3091     pIter->base.nData = pSeg->nPos;
  3096   3092   
  3097   3093     assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
  3098         -  assert( pIter->pColset==0 || pIter->bFiltered );
         3094  +  assert( pIter->pColset==0 );
  3099   3095   
  3100   3096     if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
  3101   3097       /* All data is stored on the current page. Populate the output 
  3102   3098       ** variables to point into the body of the page object. */
  3103   3099       pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  3104   3100     }else{
  3105   3101       /* The data is distributed over two or more pages. Copy it into the
................................................................................
  3208   3204   static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
  3209   3205     if( *pRc==SQLITE_OK ){
  3210   3206       Fts5Config *pConfig = pIter->pIndex->pConfig;
  3211   3207       if( pConfig->eDetail==FTS5_DETAIL_NONE ){
  3212   3208         pIter->xSetOutputs = fts5IterSetOutputs_None;
  3213   3209       }
  3214   3210   
  3215         -    else if( pIter->pColset==0 || pIter->bFiltered ){
         3211  +    else if( pIter->pColset==0 ){
  3216   3212         pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
  3217   3213       }
  3218   3214   
  3219   3215       else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
  3220   3216         pIter->xSetOutputs = fts5IterSetOutputs_Full;
  3221   3217       }
  3222   3218   
................................................................................
  3355   3351     Fts5Iter **ppOut                /* New object */
  3356   3352   ){
  3357   3353     Fts5Iter *pNew;
  3358   3354     pNew = fts5MultiIterAlloc(p, 2);
  3359   3355     if( pNew ){
  3360   3356       Fts5SegIter *pIter = &pNew->aSeg[1];
  3361   3357   
  3362         -    pNew->bFiltered = 1;
  3363   3358       pIter->flags = FTS5_SEGITER_ONETERM;
  3364   3359       if( pData->szLeaf>0 ){
  3365   3360         pIter->pLeaf = pData;
  3366   3361         pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid);
  3367   3362         pIter->iEndofDoclist = pData->nn;
  3368   3363         pNew->aFirst[1].iFirst = 1;
  3369   3364         if( bDesc ){
................................................................................
  4700   4695   ** already occurred, this function is a no-op.
  4701   4696   */
  4702   4697   static void fts5MergePrefixLists(
  4703   4698     Fts5Index *p,                   /* FTS5 backend object */
  4704   4699     Fts5Buffer *p1,                 /* First list to merge */
  4705   4700     Fts5Buffer *p2                  /* Second list to merge */
  4706   4701   ){
  4707         -
  4708   4702     if( p2->n ){
  4709         -    if( p1->n==0 ){
  4710         -      fts5BufferSwap(p1, p2);
  4711         -    }else{
  4712         -      i64 iLastRowid = 0;
  4713         -      Fts5DoclistIter i1;
  4714         -      Fts5DoclistIter i2;
  4715         -      Fts5Buffer out = {0, 0, 0};
  4716         -      Fts5Buffer tmp = {0, 0, 0};
  4717         -
  4718         -      if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
  4719         -      fts5DoclistIterInit(p1, &i1);
  4720         -      fts5DoclistIterInit(p2, &i2);
  4721         -
  4722         -      while( 1 ){
  4723         -        if( i1.iRowid<i2.iRowid ){
  4724         -          /* Copy entry from i1 */
  4725         -          fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4726         -          fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
  4727         -          fts5DoclistIterNext(&i1);
  4728         -          if( i1.aPoslist==0 ) break;
  4729         -        }
  4730         -        else if( i2.iRowid!=i1.iRowid ){
  4731         -          /* Copy entry from i2 */
  4732         -          fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4733         -          fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
  4734         -          fts5DoclistIterNext(&i2);
  4735         -          if( i2.aPoslist==0 ) break;
  4736         -        }
  4737         -        else{
  4738         -          i64 iPos1 = 0;
  4739         -          i64 iPos2 = 0;
  4740         -          int iOff1 = 0;
  4741         -          int iOff2 = 0;
  4742         -          u8 *a1 = &i1.aPoslist[i1.nSize];
  4743         -          u8 *a2 = &i2.aPoslist[i2.nSize];
  4744         -
  4745         -          i64 iPrev = 0;
  4746         -          Fts5PoslistWriter writer;
  4747         -          memset(&writer, 0, sizeof(writer));
  4748         -
  4749         -          /* Merge the two position lists. */ 
  4750         -          fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4751         -          fts5BufferZero(&tmp);
  4752         -          sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
  4753         -          if( p->rc ) break;
  4754         -
         4703  +    i64 iLastRowid = 0;
         4704  +    Fts5DoclistIter i1;
         4705  +    Fts5DoclistIter i2;
         4706  +    Fts5Buffer out = {0, 0, 0};
         4707  +    Fts5Buffer tmp = {0, 0, 0};
         4708  +
         4709  +    if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
         4710  +    fts5DoclistIterInit(p1, &i1);
         4711  +    fts5DoclistIterInit(p2, &i2);
         4712  +
         4713  +    while( 1 ){
         4714  +      if( i1.iRowid<i2.iRowid ){
         4715  +        /* Copy entry from i1 */
         4716  +        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4717  +        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize);
         4718  +        fts5DoclistIterNext(&i1);
         4719  +        if( i1.aPoslist==0 ) break;
         4720  +      }
         4721  +      else if( i2.iRowid!=i1.iRowid ){
         4722  +        /* Copy entry from i2 */
         4723  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4724  +        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize);
         4725  +        fts5DoclistIterNext(&i2);
         4726  +        if( i2.aPoslist==0 ) break;
         4727  +      }
         4728  +      else{
         4729  +        /* Merge the two position lists. */ 
         4730  +        i64 iPos1 = 0;
         4731  +        i64 iPos2 = 0;
         4732  +        int iOff1 = 0;
         4733  +        int iOff2 = 0;
         4734  +        u8 *a1 = &i1.aPoslist[i1.nSize];
         4735  +        u8 *a2 = &i2.aPoslist[i2.nSize];
         4736  +
         4737  +        i64 iPrev = 0;
         4738  +        Fts5PoslistWriter writer;
         4739  +        memset(&writer, 0, sizeof(writer));
         4740  +
         4741  +        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4742  +        fts5BufferZero(&tmp);
         4743  +        sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
         4744  +        if( p->rc ) break;
         4745  +
         4746  +        sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4747  +        sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4748  +        assert( iPos1>=0 && iPos2>=0 );
         4749  +
         4750  +        if( iPos1<iPos2 ){
         4751  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4755   4752             sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4753  +        }else{
         4754  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
  4756   4755             sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4757         -          assert( iPos1>=0 && iPos2>=0 );
         4756  +        }
  4758   4757   
  4759         -          if( iPos1<iPos2 ){
  4760         -            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4761         -            sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4762         -          }else{
  4763         -            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
  4764         -            sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4765         -          }
  4766         -
  4767         -          if( iPos1>=0 && iPos2>=0 ){
  4768         -            while( 1 ){
  4769         -              if( iPos1<iPos2 ){
  4770         -                if( iPos1!=iPrev ){
  4771         -                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4772         -                }
  4773         -                sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
  4774         -                if( iPos1<0 ) break;
  4775         -              }else{
  4776         -                if( iPos2!=iPrev ){
  4777         -                  sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
  4778         -                }
  4779         -                sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
  4780         -                if( iPos2<0 ) break;
         4758  +        if( iPos1>=0 && iPos2>=0 ){
         4759  +          while( 1 ){
         4760  +            if( iPos1<iPos2 ){
         4761  +              if( iPos1!=iPrev ){
         4762  +                sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4781   4763                 }
         4764  +              sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
         4765  +              if( iPos1<0 ) break;
         4766  +            }else{
         4767  +              assert( iPos2!=iPrev );
         4768  +              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4769  +              sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
         4770  +              if( iPos2<0 ) break;
  4782   4771               }
  4783   4772             }
         4773  +        }
  4784   4774   
  4785         -          if( iPos1>=0 ){
  4786         -            if( iPos1!=iPrev ){
  4787         -              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4788         -            }
  4789         -            fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
         4775  +        if( iPos1>=0 ){
         4776  +          if( iPos1!=iPrev ){
         4777  +            sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
  4790   4778             }
  4791         -          else if( iPos2>=0 ){
  4792         -            if( iPos2!=iPrev ){
  4793         -              sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
  4794         -            }
  4795         -            fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
  4796         -          }
  4797         -
  4798         -          /* WRITEPOSLISTSIZE */
  4799         -          fts5BufferSafeAppendVarint(&out, tmp.n * 2);
  4800         -          fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
  4801         -          fts5DoclistIterNext(&i1);
  4802         -          fts5DoclistIterNext(&i2);
  4803         -          if( i1.aPoslist==0 || i2.aPoslist==0 ) break;
         4779  +          fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
         4780  +        }else{
         4781  +          assert( iPos2>=0 && iPos2!=iPrev );
         4782  +          sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
         4783  +          fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
  4804   4784           }
         4785  +
         4786  +        /* WRITEPOSLISTSIZE */
         4787  +        fts5BufferSafeAppendVarint(&out, tmp.n * 2);
         4788  +        fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n);
         4789  +        fts5DoclistIterNext(&i1);
         4790  +        fts5DoclistIterNext(&i2);
         4791  +        if( i1.aPoslist==0 || i2.aPoslist==0 ) break;
  4805   4792         }
         4793  +    }
  4806   4794   
  4807         -      if( i1.aPoslist ){
  4808         -        fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
  4809         -        fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
  4810         -      }
  4811         -      else if( i2.aPoslist ){
  4812         -        fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
  4813         -        fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
  4814         -      }
         4795  +    if( i1.aPoslist ){
         4796  +      fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
         4797  +      fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist);
         4798  +    }
         4799  +    else if( i2.aPoslist ){
         4800  +      fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
         4801  +      fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist);
         4802  +    }
  4815   4803   
  4816         -     error_out:
  4817         -      fts5BufferSet(&p->rc, p1, out.n, out.p);
  4818         -      fts5BufferFree(&tmp);
  4819         -      fts5BufferFree(&out);
  4820         -    }
         4804  +    fts5BufferSet(&p->rc, p1, out.n, out.p);
         4805  +    fts5BufferFree(&tmp);
         4806  +    fts5BufferFree(&out);
  4821   4807     }
  4822   4808   }
  4823   4809   
  4824   4810   static void fts5SetupPrefixIter(
  4825   4811     Fts5Index *p,                   /* Index to read from */
  4826   4812     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4827   4813     const u8 *pToken,               /* Buffer containing prefix to match */
................................................................................
  5177   5163           fts5StructureRelease(pStruct);
  5178   5164         }
  5179   5165       }else{
  5180   5166         /* Scan multiple terms in the main index */
  5181   5167         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  5182   5168         buf.p[0] = FTS5_MAIN_PREFIX;
  5183   5169         fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
         5170  +      assert( pRet->pColset==0 );
  5184   5171         fts5IterSetOutputCb(&p->rc, pRet);
  5185   5172         if( p->rc==SQLITE_OK ){
  5186   5173           Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
  5187         -        if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
         5174  +        if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
  5188   5175         }
  5189   5176       }
  5190   5177   
  5191   5178       if( p->rc ){
  5192   5179         sqlite3Fts5IterClose(&pRet->base);
  5193   5180         pRet = 0;
  5194   5181         fts5CloseReader(p);

Changes to ext/fts5/fts5_varint.c.

   329    329       return 2;
   330    330     }
   331    331     return fts5PutVarint64(p,v);
   332    332   }
   333    333   
   334    334   
   335    335   int sqlite3Fts5GetVarintLen(u32 iVal){
          336  +#if 0
   336    337     if( iVal<(1 << 7 ) ) return 1;
          338  +#endif
          339  +  assert( iVal>=(1 << 7) );
   337    340     if( iVal<(1 << 14) ) return 2;
   338    341     if( iVal<(1 << 21) ) return 3;
   339    342     if( iVal<(1 << 28) ) return 4;
   340    343     return 5;
   341    344   }
   342    345   

Changes to ext/fts5/test/fts5config.test.

   239    239     5 {detail=',1'}
   240    240     6 {detail=''}
   241    241   } {
   242    242     set res [list 1 {malformed detail=... directive}]
   243    243     do_catchsql_test 11.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
   244    244   }
   245    245   
          246  +do_catchsql_test 12.1 {
          247  +  INSERT INTO t1(t1, rank) VALUES('rank', NULL);;
          248  +} {1 {SQL logic error or missing database}}
          249  +
   246    250   finish_test
   247    251