/ Check-in [bdb8e82a]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Have fts5 integrity check verify that prefix indexes contain the same values as returned by prefix queries on the main terms index.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: bdb8e82ab683f2438cde9f0b63e497dbf0141dcf
User & Date: dan 2015-04-11 18:25:04
Context
2015-04-14
20:15
Begin changing fts5 to use a delete flag so that delete markers may be annihilated more quickly. check-in: 9341c070 user: dan tags: fts5
2015-04-11
18:25
Have fts5 integrity check verify that prefix indexes contain the same values as returned by prefix queries on the main terms index. check-in: bdb8e82a user: dan tags: fts5
16:23
Improve fts5 integrity-check so that it checks that DESC queries return the same as ASC. Change the poslist format slightly to make room for a delete-flag. check-in: 49c1e745 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   223    223   
   224    224   typedef struct Fts5Index Fts5Index;
   225    225   typedef struct Fts5IndexIter Fts5IndexIter;
   226    226   
   227    227   /*
   228    228   ** Values used as part of the flags argument passed to IndexQuery().
   229    229   */
   230         -#define FTS5INDEX_QUERY_PREFIX  0x0001      /* Prefix query */
   231         -#define FTS5INDEX_QUERY_DESC    0x0002      /* Docs in descending rowid order */
          230  +#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
          231  +#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
          232  +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
   232    233   
   233    234   /*
   234    235   ** Create/destroy an Fts5Index object.
   235    236   */
   236    237   int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**);
   237    238   int sqlite3Fts5IndexClose(Fts5Index *p, int bDestroy);
   238    239   

Changes to ext/fts5/fts5_index.c.

  4424   4424           u64 ck1 = 0;
  4425   4425           u64 ck2 = 0;
  4426   4426   
  4427   4427           /* Check that the results returned for ASC and DESC queries are
  4428   4428           ** the same. If not, call this corruption.  */
  4429   4429           rc = fts5QueryCksum(p, z, n, flags, &ck1);
  4430   4430           if( rc==SQLITE_OK ){
  4431         -          rc = fts5QueryCksum(p, z, n, flags | FTS5INDEX_QUERY_DESC, &ck2);
         4431  +          rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2);
  4432   4432           }
  4433   4433           if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4434  +
         4435  +        /* If this is a prefix query, check that the results returned if the
         4436  +        ** the index is disabled are the same. In both ASC and DESC order. */
         4437  +        if( iIdx>0 && rc==SQLITE_OK ){
         4438  +          int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
         4439  +          ck2 = 0;
         4440  +          rc = fts5QueryCksum(p, z, n, f, &ck2);
         4441  +          if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4442  +        }
         4443  +        if( iIdx>0 && rc==SQLITE_OK ){
         4444  +          int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
         4445  +          ck2 = 0;
         4446  +          rc = fts5QueryCksum(p, z, n, f, &ck2);
         4447  +          if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4448  +        }
  4434   4449   
  4435   4450           cksum3 ^= ck1;
  4436   4451           fts5BufferSet(&rc, &term, n, (const u8*)z);
  4437   4452           p->rc = rc;
  4438   4453         }
  4439   4454       }
  4440   4455       fts5MultiIterFree(p, pIter);
................................................................................
  4702   4717   */
  4703   4718   int sqlite3Fts5IndexQuery(
  4704   4719     Fts5Index *p,                   /* FTS index to query */
  4705   4720     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4706   4721     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  4707   4722     Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4708   4723   ){
         4724  +  Fts5Config *pConfig = p->pConfig;
  4709   4725     Fts5IndexIter *pRet;
  4710   4726     int iIdx = 0;
  4711   4727   
  4712   4728     if( flags & FTS5INDEX_QUERY_PREFIX ){
  4713         -    Fts5Config *pConfig = p->pConfig;
  4714         -    int nChar = fts5IndexCharlen(pToken, nToken);
  4715         -    for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
  4716         -      if( pConfig->aPrefix[iIdx-1]==nChar ) break;
  4717         -    }
  4718         -    if( iIdx>pConfig->nPrefix ){
  4719         -      iIdx = -1;
         4729  +    if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
         4730  +      iIdx = 1+pConfig->nPrefix;
         4731  +    }else{
         4732  +      int nChar = fts5IndexCharlen(pToken, nToken);
         4733  +      for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
         4734  +        if( pConfig->aPrefix[iIdx-1]==nChar ) break;
         4735  +      }
  4720   4736       }
  4721   4737     }
  4722   4738   
  4723   4739     pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
  4724   4740     if( pRet ){
  4725   4741       memset(pRet, 0, sizeof(Fts5IndexIter));
  4726   4742   
  4727   4743       pRet->pIndex = p;
  4728         -    if( iIdx>=0 ){
         4744  +    if( iIdx<=pConfig->nPrefix ){
  4729   4745         pRet->pStruct = fts5StructureRead(p, iIdx);
  4730   4746         if( pRet->pStruct ){
  4731   4747           fts5MultiIterNew(p, pRet->pStruct, 
  4732   4748               iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
  4733   4749           );
  4734   4750         }
  4735   4751       }else{