/ Check-in [ac8f4cf0]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Updates to fts5 to support detail=none mode. As of this commit, many cases are still broken.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1: ac8f4cf0cede6bcbe47eeefb85d80a27e9278212
User & Date: dan 2015-12-30 19:58:57
Context
2015-12-31
17:36
Fix some problems with fts5 detail=none tables. Some still remain. check-in: 6a6f7bc4 user: dan tags: fts5-offsets
2015-12-30
19:58
Updates to fts5 to support detail=none mode. As of this commit, many cases are still broken. check-in: ac8f4cf0 user: dan tags: fts5-offsets
2015-12-29
19:35
Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase. check-in: 8c30605b user: dan tags: fts5-offsets
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5_hash.c.

    59     59     Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
    60     60     Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
    61     61     
    62     62     int nAlloc;                     /* Total size of allocation */
    63     63     int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
    64     64     int nData;                      /* Total bytes of data (incl. structure) */
    65     65     u8 bDel;                        /* Set delete-flag @ iSzPoslist */
           66  +  u8 bContent;                    /* Set content-flag (detail=none mode) */
    66     67   
    67     68     int iCol;                       /* Column of last value written */
    68     69     int iPos;                       /* Position of last value written */
    69     70     i64 iRowid;                     /* Rowid of last value written */
    70     71     char zKey[8];                   /* Nul-terminated entry key */
    71     72   };
    72     73   
................................................................................
   180    181   
   181    182     sqlite3_free(apOld);
   182    183     pHash->nSlot = nNew;
   183    184     pHash->aSlot = apNew;
   184    185     return SQLITE_OK;
   185    186   }
   186    187   
   187         -static void fts5HashAddPoslistSize(Fts5HashEntry *p){
          188  +static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
   188    189     if( p->iSzPoslist ){
   189    190       u8 *pPtr = (u8*)p;
   190         -    int nSz = (p->nData - p->iSzPoslist - 1);         /* Size in bytes */
   191         -    int nPos = nSz*2 + p->bDel;                       /* Value of nPos field */
   192         -
   193         -    assert( p->bDel==0 || p->bDel==1 );
   194         -    if( nPos<=127 ){
   195         -      pPtr[p->iSzPoslist] = (u8)nPos;
          191  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          192  +      assert( p->nData==p->iSzPoslist );
          193  +      if( p->bDel ){
          194  +        pPtr[p->nData++] = 0x00;
          195  +        if( p->bContent ){
          196  +          pPtr[p->nData++] = 0x00;
          197  +        }
          198  +      }
   196    199       }else{
   197         -      int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
   198         -      memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
   199         -      sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
   200         -      p->nData += (nByte-1);
          200  +      int nSz = (p->nData - p->iSzPoslist - 1);       /* Size in bytes */
          201  +      int nPos = nSz*2 + p->bDel;                     /* Value of nPos field */
          202  +
          203  +      assert( p->bDel==0 || p->bDel==1 );
          204  +      if( nPos<=127 ){
          205  +        pPtr[p->iSzPoslist] = (u8)nPos;
          206  +      }else{
          207  +        int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
          208  +        memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
          209  +        sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
          210  +        p->nData += (nByte-1);
          211  +      }
   201    212       }
   202         -    p->bDel = 0;
          213  +
   203    214       p->iSzPoslist = 0;
          215  +    p->bDel = 0;
          216  +    p->bContent = 0;
   204    217     }
   205    218   }
   206    219   
          220  +/*
          221  +** Add an entry to the in-memory hash table. The key is the concatenation
          222  +** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
          223  +**
          224  +**     (bByte || pToken) -> (iRowid,iCol,iPos)
          225  +**
          226  +** Or, if iCol is negative, then the value is a delete marker.
          227  +*/
   207    228   int sqlite3Fts5HashWrite(
   208    229     Fts5Hash *pHash,
   209    230     i64 iRowid,                     /* Rowid for this entry */
   210    231     int iCol,                       /* Column token appears in (-ve -> delete) */
   211    232     int iPos,                       /* Position of token within column */
   212    233     char bByte,                     /* First byte of token */
   213    234     const char *pToken, int nToken  /* Token to add or remove to or from index */
................................................................................
   229    250       ){
   230    251         break;
   231    252       }
   232    253     }
   233    254   
   234    255     /* If an existing hash entry cannot be found, create a new one. */
   235    256     if( p==0 ){
          257  +    /* Figure out how much space to allocate */
   236    258       int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
   237    259       if( nByte<128 ) nByte = 128;
   238    260   
          261  +    /* Grow the Fts5Hash.aSlot[] array if necessary. */
   239    262       if( (pHash->nEntry*2)>=pHash->nSlot ){
   240    263         int rc = fts5HashResize(pHash);
   241    264         if( rc!=SQLITE_OK ) return rc;
   242    265         iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   243    266       }
   244    267   
          268  +    /* Allocate new Fts5HashEntry and add it to the hash table. */
   245    269       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   246    270       if( !p ) return SQLITE_NOMEM;
   247    271       memset(p, 0, FTS5_HASHENTRYSIZE);
   248    272       p->nAlloc = nByte;
   249    273       p->zKey[0] = bByte;
   250    274       memcpy(&p->zKey[1], pToken, nToken);
   251    275       assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
   252    276       p->zKey[nToken+1] = '\0';
   253    277       p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
   254         -    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
   255         -    p->iSzPoslist = p->nData;
   256         -    p->nData += 1;
   257         -    p->iRowid = iRowid;
   258         -    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
   259    278       p->pHashNext = pHash->aSlot[iHash];
   260    279       pHash->aSlot[iHash] = p;
   261    280       pHash->nEntry++;
          281  +
          282  +    /* Add the first rowid field to the hash-entry */
          283  +    p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
          284  +    p->iRowid = iRowid;
          285  +
          286  +    p->iSzPoslist = p->nData;
          287  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          288  +      p->nData += 1;
          289  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          290  +    }
          291  +
   262    292       nIncr += p->nData;
   263         -  }
   264         -
   265         -  /* Check there is enough space to append a new entry. Worst case scenario
   266         -  ** is:
   267         -  **
   268         -  **     + 9 bytes for a new rowid,
   269         -  **     + 4 byte reserved for the "poslist size" varint.
   270         -  **     + 1 byte for a "new column" byte,
   271         -  **     + 3 bytes for a new column number (16-bit max) as a varint,
   272         -  **     + 5 bytes for the new position offset (32-bit max).
   273         -  */
   274         -  if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
   275         -    int nNew = p->nAlloc * 2;
   276         -    Fts5HashEntry *pNew;
   277         -    Fts5HashEntry **pp;
   278         -    pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
   279         -    if( pNew==0 ) return SQLITE_NOMEM;
   280         -    pNew->nAlloc = nNew;
   281         -    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
   282         -    *pp = pNew;
   283         -    p = pNew;
          293  +  }else{
          294  +
          295  +    /* Appending to an existing hash-entry. Check that there is enough 
          296  +    ** space to append the largest possible new entry. Worst case scenario 
          297  +    ** is:
          298  +    **
          299  +    **     + 9 bytes for a new rowid,
          300  +    **     + 4 byte reserved for the "poslist size" varint.
          301  +    **     + 1 byte for a "new column" byte,
          302  +    **     + 3 bytes for a new column number (16-bit max) as a varint,
          303  +    **     + 5 bytes for the new position offset (32-bit max).
          304  +    */
          305  +    if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
          306  +      int nNew = p->nAlloc * 2;
          307  +      Fts5HashEntry *pNew;
          308  +      Fts5HashEntry **pp;
          309  +      pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
          310  +      if( pNew==0 ) return SQLITE_NOMEM;
          311  +      pNew->nAlloc = nNew;
          312  +      for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
          313  +      *pp = pNew;
          314  +      p = pNew;
          315  +    }
          316  +    nIncr -= p->nData;
   284    317     }
          318  +  assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
          319  +
   285    320     pPtr = (u8*)p;
   286         -  nIncr -= p->nData;
   287    321   
   288    322     /* If this is a new rowid, append the 4-byte size field for the previous
   289    323     ** entry, and the new rowid for this entry.  */
   290    324     if( iRowid!=p->iRowid ){
   291         -    fts5HashAddPoslistSize(p);
          325  +    fts5HashAddPoslistSize(pHash, p);
   292    326       p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
   293         -    p->iSzPoslist = p->nData;
   294         -    p->nData += 1;
   295         -    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
   296         -    p->iPos = 0;
   297    327       p->iRowid = iRowid;
   298    328       bNew = 1;
          329  +    p->iSzPoslist = p->nData;
          330  +    if( pHash->eDetail!=FTS5_DETAIL_NONE ){
          331  +      p->nData += 1;
          332  +      p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
          333  +      p->iPos = 0;
          334  +    }
   299    335     }
   300    336   
   301    337     if( iCol>=0 ){
   302         -    /* Append a new column value, if necessary */
   303         -    assert( iCol>=p->iCol );
   304         -    if( iCol!=p->iCol ){
   305         -      if( pHash->eDetail==FTS5_DETAIL_FULL ){
   306         -        pPtr[p->nData++] = 0x01;
   307         -        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
   308         -        p->iCol = iCol;
   309         -        p->iPos = 0;
   310         -      }else{
   311         -        bNew = 1;
   312         -        p->iCol = iPos = iCol;
          338  +    if( pHash->eDetail==FTS5_DETAIL_NONE ){
          339  +      p->bContent = 1;
          340  +    }else{
          341  +      /* Append a new column value, if necessary */
          342  +      assert( iCol>=p->iCol );
          343  +      if( iCol!=p->iCol ){
          344  +        if( pHash->eDetail==FTS5_DETAIL_FULL ){
          345  +          pPtr[p->nData++] = 0x01;
          346  +          p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
          347  +          p->iCol = iCol;
          348  +          p->iPos = 0;
          349  +        }else{
          350  +          bNew = 1;
          351  +          p->iCol = iPos = iCol;
          352  +        }
   313    353         }
   314         -    }
   315    354   
   316         -    /* Append the new position offset, if necessary */
   317         -    if( bNew ){
   318         -      p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
   319         -      p->iPos = iPos;
          355  +      /* Append the new position offset, if necessary */
          356  +      if( bNew ){
          357  +        p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
          358  +        p->iPos = iPos;
          359  +      }
   320    360       }
   321    361     }else{
   322    362       /* This is a delete. Set the delete flag. */
   323    363       p->bDel = 1;
   324    364     }
          365  +
   325    366     nIncr += p->nData;
   326         -
   327    367     *pHash->pnByte += nIncr;
   328    368     return SQLITE_OK;
   329    369   }
   330    370   
   331    371   
   332    372   /*
   333    373   ** Arguments pLeft and pRight point to linked-lists of hash-entry objects,
................................................................................
   433    473     Fts5HashEntry *p;
   434    474   
   435    475     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   436    476       if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
   437    477     }
   438    478   
   439    479     if( p ){
   440         -    fts5HashAddPoslistSize(p);
          480  +    fts5HashAddPoslistSize(pHash, p);
   441    481       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   442    482       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   443    483     }else{
   444    484       *ppDoclist = 0;
   445    485       *pnDoclist = 0;
   446    486     }
   447    487   
................................................................................
   469    509     const char **pzTerm,            /* OUT: term (nul-terminated) */
   470    510     const u8 **ppDoclist,           /* OUT: pointer to doclist */
   471    511     int *pnDoclist                  /* OUT: size of doclist in bytes */
   472    512   ){
   473    513     Fts5HashEntry *p;
   474    514     if( (p = pHash->pScan) ){
   475    515       int nTerm = (int)strlen(p->zKey);
   476         -    fts5HashAddPoslistSize(p);
          516  +    fts5HashAddPoslistSize(pHash, p);
   477    517       *pzTerm = p->zKey;
   478    518       *ppDoclist = (const u8*)&p->zKey[nTerm+1];
   479    519       *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
   480    520     }else{
   481    521       *pzTerm = 0;
   482    522       *ppDoclist = 0;
   483    523       *pnDoclist = 0;
   484    524     }
   485    525   }
   486    526   

Changes to ext/fts5/fts5_index.c.

   448    448   
   449    449     Fts5DlidxIter *pDlidx;          /* If there is a doclist-index */
   450    450   
   451    451     /* Variables populated based on current entry. */
   452    452     Fts5Buffer term;                /* Current term */
   453    453     i64 iRowid;                     /* Current rowid */
   454    454     int nPos;                       /* Number of bytes in current position list */
   455         -  int bDel;                       /* True if the delete flag is set */
          455  +  u8 bDel;                        /* True if the delete flag is set */
          456  +  // u8 bContent;                    /* True if has content (detail=none mode) */
   456    457   };
   457    458   
   458    459   /*
   459    460   ** Argument is a pointer to an Fts5Data structure that contains a 
   460    461   ** leaf page.
   461    462   */
   462    463   #define ASSERT_SZLEAF_OK(x) assert( \
   463    464       (x)->szLeaf==(x)->nn || (x)->szLeaf==fts5GetU16(&(x)->p[2]) \
   464    465   )
   465    466   
   466    467   #define FTS5_SEGITER_ONETERM 0x01
   467    468   #define FTS5_SEGITER_REVERSE 0x02
   468         -
   469    469   
   470    470   /* 
   471    471   ** Argument is a pointer to an Fts5Data structure that contains a leaf
   472    472   ** page. This macro evaluates to true if the leaf contains no terms, or
   473    473   ** false if it contains at least one term.
   474    474   */
   475    475   #define fts5LeafIsTermless(x) ((x)->szLeaf >= (x)->nn)
................................................................................
  1488   1488   **
  1489   1489   ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the 
  1490   1490   ** position list content (if any).
  1491   1491   */
  1492   1492   static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
  1493   1493     if( p->rc==SQLITE_OK ){
  1494   1494       int iOff = pIter->iLeafOffset;  /* Offset to read at */
  1495         -    int nSz;
  1496   1495       ASSERT_SZLEAF_OK(pIter->pLeaf);
  1497         -    fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
  1498         -    pIter->bDel = (nSz & 0x0001);
  1499         -    pIter->nPos = nSz>>1;
         1496  +    if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1497  +      pIter->bDel = 0;
         1498  +      pIter->nPos = 1;
         1499  +      if( iOff<pIter->pLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){
         1500  +        pIter->bDel = 1;
         1501  +        iOff++;
         1502  +        if( iOff<pIter->pLeaf->szLeaf && pIter->pLeaf->p[iOff]==0 ){
         1503  +          pIter->nPos = 1;
         1504  +          iOff++;
         1505  +        }else{
         1506  +          pIter->nPos = 0;
         1507  +        }
         1508  +      }
         1509  +    }else{
         1510  +      int nSz;
         1511  +      fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz);
         1512  +      pIter->bDel = (nSz & 0x0001);
         1513  +      pIter->nPos = nSz>>1;
         1514  +      assert_nc( pIter->nPos>=0 );
         1515  +    }
  1500   1516       pIter->iLeafOffset = iOff;
  1501         -    assert_nc( pIter->nPos>=0 );
  1502   1517     }
  1503   1518   }
  1504   1519   
  1505   1520   static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
  1506   1521     u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
  1507   1522     int iOff = pIter->iLeafOffset;
  1508   1523   
................................................................................
  1754   1769         int nKeep = 0;
  1755   1770   
  1756   1771         /* Search for the end of the position list within the current page. */
  1757   1772         u8 *a = pLeaf->p;
  1758   1773         int n = pLeaf->szLeaf;
  1759   1774   
  1760   1775         ASSERT_SZLEAF_OK(pLeaf);
  1761         -      iOff = pIter->iLeafOffset + pIter->nPos;
         1776  +      if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1777  +        iOff = pIter->iLeafOffset;
         1778  +      }else{
         1779  +        iOff = pIter->iLeafOffset + pIter->nPos;
         1780  +      }
  1762   1781   
  1763   1782         if( iOff<n ){
  1764   1783           /* The next entry is on the current page. */
  1765   1784           assert_nc( iOff<=pIter->iEndofDoclist );
  1766   1785           if( iOff>=pIter->iEndofDoclist ){
  1767   1786             bNewTerm = 1;
  1768   1787             if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
................................................................................
  1842   1861               fts5SegIterLoadTerm(p, pIter, nKeep);
  1843   1862               fts5SegIterLoadNPos(p, pIter);
  1844   1863               if( pbNewTerm ) *pbNewTerm = 1;
  1845   1864             }
  1846   1865           }else{
  1847   1866             /* The following could be done by calling fts5SegIterLoadNPos(). But
  1848   1867             ** this block is particularly performance critical, so equivalent
  1849         -          ** code is inlined. */
         1868  +          ** code is inlined. 
         1869  +          **
         1870  +          ** Later: Switched back to fts5SegIterLoadNPos() because it supports
         1871  +          ** detail=none mode. Not ideal.
         1872  +          */
         1873  +#if 0
  1850   1874             int nSz;
  1851   1875             assert( p->rc==SQLITE_OK );
  1852   1876             fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz);
  1853   1877             pIter->bDel = (nSz & 0x0001);
  1854   1878             pIter->nPos = nSz>>1;
  1855   1879             assert_nc( pIter->nPos>=0 );
         1880  +#endif
         1881  +          fts5SegIterLoadNPos(p, pIter);
  1856   1882           }
  1857   1883         }
  1858   1884       }
  1859   1885     }
  1860   1886   }
  1861   1887   
  1862   1888   #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; }
................................................................................
  4161   4187     assert( (prev & 0x80)==0 );
  4162   4188     while( p<pEnd && ((prev & 0x80) || *p!=0x01) ){
  4163   4189       prev = *p++;
  4164   4190     }
  4165   4191     return p - (*pa);
  4166   4192   }
  4167   4193   
         4194  +static int fts5AppendRowid(
         4195  +  Fts5Index *p,
         4196  +  i64 iDelta,
         4197  +  Fts5IndexIter *pMulti,
         4198  +  Fts5Colset *pColset,
         4199  +  Fts5Buffer *pBuf
         4200  +){
         4201  +  fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
         4202  +  return 0;
         4203  +}
  4168   4204   
  4169   4205   /*
  4170   4206   ** Iterator pMulti currently points to a valid entry (not EOF). This
  4171   4207   ** function appends the following to buffer pBuf:
  4172   4208   **
  4173   4209   **   * The varint iDelta, and
  4174   4210   **   * the position list that currently points to, including the size field.
................................................................................
  4300   4336   #endif
  4301   4337   
  4302   4338   #define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) {       \
  4303   4339     assert( (pBuf)->n!=0 || (iLastRowid)==0 );                   \
  4304   4340     fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \
  4305   4341     (iLastRowid) = (iRowid);                                     \
  4306   4342   }
         4343  +
         4344  +/*
         4345  +** Swap the contents of buffer *p1 with that of *p2.
         4346  +*/
         4347  +static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
         4348  +  Fts5Buffer tmp = *p1;
         4349  +  *p1 = *p2;
         4350  +  *p2 = tmp;
         4351  +}
         4352  +
         4353  +static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){
         4354  +  int i = *piOff;
         4355  +  if( i>=pBuf->n ){
         4356  +    *piOff = -1;
         4357  +  }else{
         4358  +    u64 iVal;
         4359  +    *piOff = i + sqlite3Fts5GetVarint(&pBuf->p[i], &iVal);
         4360  +    *piRowid += iVal;
         4361  +  }
         4362  +}
         4363  +
         4364  +/*
         4365  +** This is the equivalent of fts5MergePrefixLists() for detail=none mode.
         4366  +** In this case the buffers consist of a delta-encoded list of rowids only.
         4367  +*/
         4368  +static void fts5MergeRowidLists(
         4369  +  Fts5Index *p,                   /* FTS5 backend object */
         4370  +  Fts5Buffer *p1,                 /* First list to merge */
         4371  +  Fts5Buffer *p2                  /* Second list to merge */
         4372  +){
         4373  +  int i1 = 0;
         4374  +  int i2 = 0;
         4375  +  i64 iRowid1 = 0;
         4376  +  i64 iRowid2 = 0;
         4377  +  i64 iOut = 0;
         4378  +
         4379  +  Fts5Buffer out;
         4380  +  memset(&out, 0, sizeof(out));
         4381  +  sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n);
         4382  +  if( p->rc ) return;
         4383  +
         4384  +  fts5NextRowid(p1, &i1, &iRowid1);
         4385  +  fts5NextRowid(p1, &i2, &iRowid2);
         4386  +  while( i1>=0 || i2>=0 ){
         4387  +    if( i1>=0 && (i2<0 || iRowid1<iRowid2) ){
         4388  +      fts5BufferSafeAppendVarint(&out, iRowid1 - iOut);
         4389  +      iOut = iRowid1;
         4390  +      fts5NextRowid(p1, &i1, &iRowid1);
         4391  +    }else{
         4392  +      fts5BufferSafeAppendVarint(&out, iRowid2 - iOut);
         4393  +      iOut = iRowid2;
         4394  +      fts5NextRowid(p2, &i2, &iRowid2);
         4395  +      if( i1>=0 && iRowid1==iRowid2 ){
         4396  +        fts5NextRowid(p1, &i1, &iRowid1);
         4397  +      }
         4398  +    }
         4399  +  }
         4400  +
         4401  +  fts5BufferSwap(&out, p1);
         4402  +  fts5BufferFree(&out);
         4403  +}
  4307   4404   
  4308   4405   /*
  4309   4406   ** Buffers p1 and p2 contain doclists. This function merges the content
  4310   4407   ** of the two doclists together and sets buffer p1 to the result before
  4311   4408   ** returning.
  4312   4409   **
  4313   4410   ** If an error occurs, an error code is left in p->rc. If an error has
................................................................................
  4388   4485   
  4389   4486       fts5BufferSet(&p->rc, p1, out.n, out.p);
  4390   4487       fts5BufferFree(&tmp);
  4391   4488       fts5BufferFree(&out);
  4392   4489     }
  4393   4490   }
  4394   4491   
  4395         -/*
  4396         -** Swap the contents of buffer *p1 with that of *p2.
  4397         -*/
  4398         -static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
  4399         -  Fts5Buffer tmp = *p1;
  4400         -  *p1 = *p2;
  4401         -  *p2 = tmp;
  4402         -}
  4403         -
  4404   4492   static void fts5SetupPrefixIter(
  4405   4493     Fts5Index *p,                   /* Index to read from */
  4406   4494     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4407   4495     const u8 *pToken,               /* Buffer containing prefix to match */
  4408   4496     int nToken,                     /* Size of buffer pToken in bytes */
  4409   4497     Fts5Colset *pColset,            /* Restrict matches to these columns */
  4410   4498     Fts5IndexIter **ppIter          /* OUT: New iterator */
  4411   4499   ){
  4412   4500     Fts5Structure *pStruct;
  4413   4501     Fts5Buffer *aBuf;
  4414   4502     const int nBuf = 32;
         4503  +
         4504  +  void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
         4505  +  int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
         4506  +  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
         4507  +    xMerge = fts5MergeRowidLists;
         4508  +    xAppend = fts5AppendRowid;
         4509  +  }else{
         4510  +    xMerge = fts5MergePrefixLists;
         4511  +    xAppend = fts5AppendPoslist;
         4512  +  }
  4415   4513   
  4416   4514     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4417   4515     pStruct = fts5StructureRead(p);
  4418   4516   
  4419   4517     if( aBuf && pStruct ){
  4420   4518       const int flags = FTS5INDEX_QUERY_SCAN;
  4421   4519       int i;
................................................................................
  4441   4539         if( doclist.n>0 && iRowid<=iLastRowid ){
  4442   4540           for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
  4443   4541             assert( i<nBuf );
  4444   4542             if( aBuf[i].n==0 ){
  4445   4543               fts5BufferSwap(&doclist, &aBuf[i]);
  4446   4544               fts5BufferZero(&doclist);
  4447   4545             }else{
  4448         -            fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4546  +            xMerge(p, &doclist, &aBuf[i]);
  4449   4547               fts5BufferZero(&aBuf[i]);
  4450   4548             }
  4451   4549           }
  4452   4550           iLastRowid = 0;
  4453   4551         }
  4454   4552   
  4455         -      if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
         4553  +      if( !xAppend(p, iRowid-iLastRowid, p1, pColset, &doclist) ){
  4456   4554           iLastRowid = iRowid;
  4457   4555         }
  4458   4556       }
  4459   4557   
  4460   4558       for(i=0; i<nBuf; i++){
  4461   4559         if( p->rc==SQLITE_OK ){
  4462         -        fts5MergePrefixLists(p, &doclist, &aBuf[i]);
         4560  +        xMerge(p, &doclist, &aBuf[i]);
  4463   4561         }
  4464   4562         fts5BufferFree(&aBuf[i]);
  4465   4563       }
  4466   4564       fts5MultiIterFree(p, p1);
  4467   4565   
  4468   4566       pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n);
  4469   4567       if( pData ){
................................................................................
  4850   4948     i64 *piRowid                    /* OUT: Current rowid */
  4851   4949   ){
  4852   4950     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  4853   4951     int eDetail = pIter->pIndex->pConfig->eDetail;
  4854   4952   
  4855   4953     assert( pIter->pIndex->rc==SQLITE_OK );
  4856   4954     *piRowid = pSeg->iRowid;
         4955  +  if( eDetail==FTS5_DETAIL_NONE ){
         4956  +    *pn = pSeg->nPos;
         4957  +  }else
  4857   4958     if( eDetail==FTS5_DETAIL_FULL 
  4858   4959      && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4859   4960     ){
  4860   4961       u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4861   4962       if( pColset==0 || pIter->bFiltered ){
  4862   4963         *pn = pSeg->nPos;
  4863   4964         *pp = pPos;

Added ext/fts5/test/fts5simple2.test.

            1  +# 2015 September 05
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5simple2
           15  +
           16  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           17  +ifcapable !fts5 {
           18  +  finish_test
           19  +  return
           20  +}
           21  +
           22  +do_execsql_test 1.0 {
           23  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           24  +  INSERT INTO t1 VALUES('a b c');
           25  +}
           26  +do_execsql_test 1.1 {
           27  +  SELECT rowid FROM t1('c a b')
           28  +} {1}
           29  +
           30  +#-------------------------------------------------------------------------
           31  +#
           32  +reset_db
           33  +do_execsql_test 2.0 {
           34  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           35  +  BEGIN;
           36  +    INSERT INTO t1 VALUES('b c d');
           37  +    INSERT INTO t1 VALUES('b c d');
           38  +  COMMIT;
           39  +}
           40  +do_execsql_test 2.1 {
           41  +  SELECT rowid FROM t1('b c d')
           42  +} {1 2}
           43  +
           44  +#-------------------------------------------------------------------------
           45  +#
           46  +reset_db
           47  +do_execsql_test 3.0 {
           48  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           49  +  BEGIN;
           50  +    INSERT INTO t1 VALUES('b c d');
           51  +    INSERT INTO t1 VALUES('b c d');
           52  +}
           53  +do_execsql_test 3.1 {
           54  +  SELECT rowid FROM t1('b c d'); COMMIT;
           55  +} {1 2}
           56  +
           57  +#-------------------------------------------------------------------------
           58  +#
           59  +reset_db
           60  +do_execsql_test 4.0 {
           61  +  CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
           62  +  BEGIN;
           63  +    INSERT INTO t1 VALUES('a1 b1 c1');
           64  +    INSERT INTO t1 VALUES('a2 b2 c2');
           65  +    INSERT INTO t1 VALUES('a3 b3 c3');
           66  +  COMMIT;
           67  +}
           68  +breakpoint
           69  +do_execsql_test 4.1 {
           70  +  SELECT rowid FROM t1('b*');
           71  +} {1 2 3}
           72  +
           73  +
           74  +finish_test
           75  +