/ Check-in [228b4d10]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5-offsets
Files: files | file ages | folders
SHA1:228b4d10e38f7d70e1b008c3c9b4a1ae3e32e30d
User & Date: dan 2015-12-28 19:55:00
Context
2015-12-29
19:35
Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase. check-in: 8c30605b user: dan tags: fts5-offsets
2015-12-28
19:55
Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. check-in: 228b4d10 user: dan tags: fts5-offsets
2015-12-22
18:54
Updates so that fts5 API functions xInst, xPhraseFirst and xPhraseNext work with the offsets=0 option. check-in: 69bffc16 user: dan tags: fts5-offsets
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   147    147     u8 *abUnindexed;                /* True for unindexed columns */
   148    148     int nPrefix;                    /* Number of prefix indexes */
   149    149     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   150    150     int eContent;                   /* An FTS5_CONTENT value */
   151    151     char *zContent;                 /* content table */ 
   152    152     char *zContentRowid;            /* "content_rowid=" option value */ 
   153    153     int bColumnsize;                /* "columnsize=" option value (dflt==1) */
   154         -  int bOffsets;                   /* "offsets=" option value (dflt==1) */
          154  +  int eDetail;                    /* FTS5_DETAIL_XXX value */
   155    155     char *zContentExprlist;
   156    156     Fts5Tokenizer *pTok;
   157    157     fts5_tokenizer *pTokApi;
   158    158   
   159    159     /* Values loaded from the %_config table */
   160    160     int iCookie;                    /* Incremented when %_config is modified */
   161    161     int pgsz;                       /* Approximate page size used in %_data */
................................................................................
   176    176   /* Current expected value of %_config table 'version' field */
   177    177   #define FTS5_CURRENT_VERSION 4
   178    178   
   179    179   #define FTS5_CONTENT_NORMAL   0
   180    180   #define FTS5_CONTENT_NONE     1
   181    181   #define FTS5_CONTENT_EXTERNAL 2
   182    182   
          183  +#define FTS5_DETAIL_FULL    0
          184  +#define FTS5_DETAIL_NONE    1
          185  +#define FTS5_DETAIL_COLUMNS 2
   183    186   
   184    187   
   185    188   
   186    189   int sqlite3Fts5ConfigParse(
   187    190       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
   188    191   );
   189    192   void sqlite3Fts5ConfigFree(Fts5Config*);
................................................................................
   632    635   
   633    636   /* Called during startup to register a UDF with SQLite */
   634    637   int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   635    638   
   636    639   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   637    640   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   638    641   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
          642  +
          643  +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*);
          644  +int sqlite3Fts5ExprPopulatePoslists(
          645  +    Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int
          646  +);
   639    647   
   640    648   int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
   641    649   
   642    650   /*******************************************
   643    651   ** The fts5_expr.c API above this point is used by the other hand-written
   644    652   ** C code in this module. The interfaces below this point are called by
   645    653   ** the parser code in fts5parse.y.  */

Changes to ext/fts5/fts5_config.c.

   190    190     assert( 0==fts5_iswhitespace(z[0]) );
   191    191     quote = z[0];
   192    192     if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
   193    193       fts5Dequote(z);
   194    194     }
   195    195   }
   196    196   
          197  +
          198  +struct Fts5Enum {
          199  +  const char *zName;
          200  +  int eVal;
          201  +};
          202  +typedef struct Fts5Enum Fts5Enum;
          203  +
          204  +static int fts5ConfigSetEnum(
          205  +  const Fts5Enum *aEnum, 
          206  +  const char *zEnum, 
          207  +  int *peVal
          208  +){
          209  +  int nEnum = strlen(zEnum);
          210  +  int i;
          211  +  int iVal = -1;
          212  +
          213  +  for(i=0; aEnum[i].zName; i++){
          214  +    if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
          215  +      if( iVal>=0 ) return SQLITE_ERROR;
          216  +      iVal = aEnum[i].eVal;
          217  +    }
          218  +  }
          219  +
          220  +  *peVal = iVal;
          221  +  return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
          222  +}
          223  +
   197    224   /*
   198    225   ** Parse a "special" CREATE VIRTUAL TABLE directive and update
   199    226   ** configuration object pConfig as appropriate.
   200    227   **
   201    228   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   202    229   ** an error occurs, an SQLite error code is returned and an error message
   203    230   ** may be left in *pzErr. It is the responsibility of the caller to
................................................................................
   340    367         rc = SQLITE_ERROR;
   341    368       }else{
   342    369         pConfig->bColumnsize = (zArg[0]=='1');
   343    370       }
   344    371       return rc;
   345    372     }
   346    373   
   347         -  if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){
   348         -    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
   349         -      *pzErr = sqlite3_mprintf("malformed offsets=... directive");
   350         -      rc = SQLITE_ERROR;
   351         -    }else{
   352         -      pConfig->bOffsets = (zArg[0]=='1');
          374  +  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
          375  +    const Fts5Enum aDetail[] = {
          376  +      { "none", FTS5_DETAIL_NONE },
          377  +      { "full", FTS5_DETAIL_FULL },
          378  +      { "columns", FTS5_DETAIL_COLUMNS },
          379  +      { 0, 0 }
          380  +    };
          381  +
          382  +    if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){
          383  +      *pzErr = sqlite3_mprintf("malformed detail=... directive");
   353    384       }
   354    385       return rc;
   355    386     }
   356    387   
   357    388     *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   358    389     return SQLITE_ERROR;
   359    390   }
................................................................................
   505    536   
   506    537     nByte = nArg * (sizeof(char*) + sizeof(u8));
   507    538     pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
   508    539     pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
   509    540     pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
   510    541     pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
   511    542     pRet->bColumnsize = 1;
   512         -  pRet->bOffsets = 1;
          543  +  pRet->eDetail = FTS5_DETAIL_FULL;
   513    544   #ifdef SQLITE_DEBUG
   514    545     pRet->bPrefixIndex = 1;
   515    546   #endif
   516    547     if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   517    548       *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   518    549       rc = SQLITE_ERROR;
   519    550     }

Changes to ext/fts5/fts5_expr.c.

  1740   1740   }
  1741   1741   
  1742   1742   void sqlite3Fts5ParseSetColset(
  1743   1743     Fts5Parse *pParse, 
  1744   1744     Fts5ExprNearset *pNear, 
  1745   1745     Fts5Colset *pColset 
  1746   1746   ){
         1747  +  if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
         1748  +    pParse->rc = SQLITE_ERROR;
         1749  +    pParse->zErr = sqlite3_mprintf(
         1750  +      "fts5: column queries are not supported (detail=none)"
         1751  +    );
         1752  +    sqlite3_free(pColset);
         1753  +    return;
         1754  +  }
         1755  +
  1747   1756     if( pNear ){
  1748   1757       pNear->pColset = pColset;
  1749   1758     }else{
  1750   1759       sqlite3_free(pColset);
  1751   1760     }
  1752   1761   }
  1753   1762   
................................................................................
  1805   1814           for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
  1806   1815             pNear->apPhrase[iPhrase]->pNode = pRet;
  1807   1816           }
  1808   1817           if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
  1809   1818             if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
  1810   1819               pRet->eType = FTS5_TERM;
  1811   1820             }
  1812         -        }else if( pParse->pConfig->bOffsets==0 ){
         1821  +        }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
  1813   1822             assert( pParse->rc==SQLITE_OK );
  1814   1823             pParse->rc = SQLITE_ERROR;
  1815   1824             assert( pParse->zErr==0 );
  1816   1825             pParse->zErr = sqlite3_mprintf(
  1817         -              "fts5: %s queries are not supported (offsets=0)", 
         1826  +              "fts5: %s queries are not supported (detail!=full)", 
  1818   1827                 pNear->nPhrase==1 ? "phrase": "NEAR"
  1819   1828             );
  1820   1829             sqlite3_free(pRet);
  1821   1830             pRet = 0;
  1822   1831           }
  1823   1832         }else{
  1824   1833           fts5ExprAddChildren(pRet, pLeft);
................................................................................
  1928   1937       for(i=0; i<pNear->nPhrase; i++){
  1929   1938         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
  1930   1939   
  1931   1940         zRet = fts5PrintfAppend(zRet, " {");
  1932   1941         for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
  1933   1942           char *zTerm = pPhrase->aTerm[iTerm].zTerm;
  1934   1943           zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
         1944  +        if( pPhrase->aTerm[iTerm].bPrefix ){
         1945  +          zRet = fts5PrintfAppend(zRet, "*");
         1946  +        }
  1935   1947         }
  1936   1948   
  1937   1949         if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
  1938   1950         if( zRet==0 ) return 0;
  1939   1951       }
  1940   1952   
  1941   1953     }else{
................................................................................
  2240   2252       nRet = pPhrase->poslist.n;
  2241   2253     }else{
  2242   2254       *pa = 0;
  2243   2255       nRet = 0;
  2244   2256     }
  2245   2257     return nRet;
  2246   2258   }
         2259  +
         2260  +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){
         2261  +  int i;
         2262  +  Fts5PoslistWriter *pRet;
         2263  +  for(i=0; i<pExpr->nPhrase; i++){
         2264  +    Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
         2265  +    assert( pExpr->apExprPhrase[i]->nTerm==1 );
         2266  +    pBuf->n = 0;
         2267  +  }
         2268  +  pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
         2269  +  if( pRet ){
         2270  +    memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase);
         2271  +  }
         2272  +  return pRet;
         2273  +}
         2274  +
         2275  +struct Fts5ExprCtx {
         2276  +  Fts5Expr *pExpr;
         2277  +  Fts5PoslistWriter *aWriter;
         2278  +  i64 iOff;
         2279  +};
         2280  +typedef struct Fts5ExprCtx Fts5ExprCtx;
         2281  +
         2282  +static int fts5ExprPopulatePoslistsCb(
         2283  +  void *pCtx,                /* Copy of 2nd argument to xTokenize() */
         2284  +  int tflags,                /* Mask of FTS5_TOKEN_* flags */
         2285  +  const char *pToken,        /* Pointer to buffer containing token */
         2286  +  int nToken,                /* Size of token in bytes */
         2287  +  int iStart,                /* Byte offset of token within input text */
         2288  +  int iEnd                   /* Byte offset of end of token within input text */
         2289  +){
         2290  +  Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
         2291  +  Fts5Expr *pExpr = p->pExpr;
         2292  +  int i;
         2293  +
         2294  +  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
         2295  +  for(i=0; i<pExpr->nPhrase; i++){
         2296  +    Fts5ExprTerm *pTerm;
         2297  +    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
         2298  +      int nTerm = strlen(pTerm->zTerm);
         2299  +      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
         2300  +       && memcmp(pTerm->zTerm, pToken, nTerm)==0
         2301  +      ){
         2302  +        int rc = sqlite3Fts5PoslistWriterAppend(
         2303  +            &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff
         2304  +        );
         2305  +        if( rc ) return rc;
         2306  +        break;
         2307  +      }
         2308  +    }
         2309  +  }
         2310  +  return SQLITE_OK;
         2311  +}
         2312  +
         2313  +int sqlite3Fts5ExprPopulatePoslists(
         2314  +  Fts5Config *pConfig,
         2315  +  Fts5Expr *pExpr, 
         2316  +  Fts5PoslistWriter *aWriter,
         2317  +  int iCol, 
         2318  +  const char *z, int n
         2319  +){
         2320  +  Fts5ExprCtx sCtx;
         2321  +  sCtx.pExpr = pExpr;
         2322  +  sCtx.aWriter = aWriter;
         2323  +  sCtx.iOff = (((i64)iCol) << 32) - 1;
         2324  +
         2325  +  return sqlite3Fts5Tokenize(pConfig, 
         2326  +      FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
         2327  +  );
         2328  +}
         2329  +

Changes to ext/fts5/fts5_hash.c.

    22     22   ** This file contains the implementation of an in-memory hash table used
    23     23   ** to accumuluate "term -> doclist" content before it is flused to a level-0
    24     24   ** segment.
    25     25   */
    26     26   
    27     27   
    28     28   struct Fts5Hash {
    29         -  int bOffsets;                   /* Copy of Fts5Config.bOffsets */
           29  +  int eDetail;                    /* Copy of Fts5Config.eDetail */
    30     30     int *pnByte;                    /* Pointer to bytes counter */
    31     31     int nEntry;                     /* Number of entries currently in hash */
    32     32     int nSlot;                      /* Size of aSlot[] array */
    33     33     Fts5HashEntry *pScan;           /* Current ordered scan item */
    34     34     Fts5HashEntry **aSlot;          /* Array of hash slots */
    35     35   };
    36     36   
................................................................................
    87     87     *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash));
    88     88     if( pNew==0 ){
    89     89       rc = SQLITE_NOMEM;
    90     90     }else{
    91     91       int nByte;
    92     92       memset(pNew, 0, sizeof(Fts5Hash));
    93     93       pNew->pnByte = pnByte;
    94         -    pNew->bOffsets = pConfig->bOffsets;
           94  +    pNew->eDetail = pConfig->eDetail;
    95     95   
    96     96       pNew->nSlot = 1024;
    97     97       nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
    98     98       pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte);
    99     99       if( pNew->aSlot==0 ){
   100    100         sqlite3_free(pNew);
   101    101         *ppNew = 0;
................................................................................
   212    212     char bByte,                     /* First byte of token */
   213    213     const char *pToken, int nToken  /* Token to add or remove to or from index */
   214    214   ){
   215    215     unsigned int iHash;
   216    216     Fts5HashEntry *p;
   217    217     u8 *pPtr;
   218    218     int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
   219         -  int bNew = pHash->bOffsets;     /* If non-delete entry should be written */
          219  +  int bNew;                       /* If non-delete entry should be written */
          220  +  
          221  +  bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
   220    222   
   221    223     /* Attempt to locate an existing hash entry */
   222    224     iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
   223    225     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   224    226       if( p->zKey[0]==bByte 
   225    227        && memcmp(&p->zKey[1], pToken, nToken)==0 
   226    228        && p->zKey[nToken+1]==0 
................................................................................
   249    251       assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
   250    252       p->zKey[nToken+1] = '\0';
   251    253       p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
   252    254       p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
   253    255       p->iSzPoslist = p->nData;
   254    256       p->nData += 1;
   255    257       p->iRowid = iRowid;
   256         -    p->iCol = (pHash->bOffsets-1);
          258  +    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
   257    259       p->pHashNext = pHash->aSlot[iHash];
   258    260       pHash->aSlot[iHash] = p;
   259    261       pHash->nEntry++;
   260    262       nIncr += p->nData;
   261    263     }
   262    264   
   263    265     /* Check there is enough space to append a new entry. Worst case scenario
................................................................................
   286    288     /* If this is a new rowid, append the 4-byte size field for the previous
   287    289     ** entry, and the new rowid for this entry.  */
   288    290     if( iRowid!=p->iRowid ){
   289    291       fts5HashAddPoslistSize(p);
   290    292       p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
   291    293       p->iSzPoslist = p->nData;
   292    294       p->nData += 1;
   293         -    p->iCol = (pHash->bOffsets-1);
          295  +    p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
   294    296       p->iPos = 0;
   295    297       p->iRowid = iRowid;
   296    298       bNew = 1;
   297    299     }
   298    300   
   299    301     if( iCol>=0 ){
   300    302       /* Append a new column value, if necessary */
   301    303       assert( iCol>=p->iCol );
   302    304       if( iCol!=p->iCol ){
   303         -      if( pHash->bOffsets==0 ){
   304         -        bNew = 1;
   305         -        p->iCol = iPos = iCol;
   306         -      }else{
          305  +      if( pHash->eDetail==FTS5_DETAIL_FULL ){
   307    306           pPtr[p->nData++] = 0x01;
   308    307           p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
   309    308           p->iCol = iCol;
   310    309           p->iPos = 0;
          310  +      }else{
          311  +        bNew = 1;
          312  +        p->iCol = iPos = iCol;
   311    313         }
   312    314       }
   313    315   
   314    316       /* Append the new position offset, if necessary */
   315    317       if( bNew ){
   316    318         p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
   317    319         p->iPos = iPos;

Changes to ext/fts5/fts5_index.c.

  4105   4105     Fts5Colset *pColset,
  4106   4106     Fts5Buffer *pBuf
  4107   4107   ){
  4108   4108     if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){
  4109   4109       if( pColset==0 ){
  4110   4110         fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback);
  4111   4111       }else{
  4112         -      if( p->pConfig->bOffsets==0 ){
  4113         -        PoslistOffsetsCtx sCtx;
  4114         -        memset(&sCtx, 0, sizeof(sCtx));
  4115         -        sCtx.pBuf = pBuf;
  4116         -        sCtx.pColset = pColset;
  4117         -        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
  4118         -      }else{
         4112  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){
  4119   4113           PoslistCallbackCtx sCtx;
  4120   4114           sCtx.pBuf = pBuf;
  4121   4115           sCtx.pColset = pColset;
  4122   4116           sCtx.eState = fts5IndexColsetTest(pColset, 0);
  4123   4117           assert( sCtx.eState==0 || sCtx.eState==1 );
  4124   4118           fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback);
         4119  +      }else{
         4120  +        PoslistOffsetsCtx sCtx;
         4121  +        memset(&sCtx, 0, sizeof(sCtx));
         4122  +        sCtx.pBuf = pBuf;
         4123  +        sCtx.pColset = pColset;
         4124  +        fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback);
  4125   4125         }
  4126   4126       }
  4127   4127     }
  4128   4128   }
  4129   4129   
  4130   4130   /*
  4131   4131   ** IN/OUT parameter (*pa) points to a position list n bytes in size. If
................................................................................
  4188   4188     Fts5Buffer *pBuf
  4189   4189   ){
  4190   4190     if( p->rc==SQLITE_OK ){
  4191   4191       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4192   4192       assert( fts5MultiIterEof(p, pMulti)==0 );
  4193   4193       assert( pSeg->nPos>0 );
  4194   4194       if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){
  4195         -      if( p->pConfig->bOffsets
         4195  +      if( p->pConfig->eDetail==FTS5_DETAIL_FULL
  4196   4196          && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4197   4197          && (pColset==0 || pColset->nCol==1)
  4198   4198         ){
  4199   4199           const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4200   4200           int nPos;
  4201   4201           if( pColset ){
  4202   4202             nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]);
................................................................................
  4846   4846     Fts5IndexIter *pIter, 
  4847   4847     Fts5Colset *pColset,            /* Column filter (or NULL) */
  4848   4848     const u8 **pp,                  /* OUT: Pointer to position-list data */
  4849   4849     int *pn,                        /* OUT: Size of position-list in bytes */
  4850   4850     i64 *piRowid                    /* OUT: Current rowid */
  4851   4851   ){
  4852   4852     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
         4853  +  int eDetail = pIter->pIndex->pConfig->eDetail;
         4854  +
  4853   4855     assert( pIter->pIndex->rc==SQLITE_OK );
  4854   4856     *piRowid = pSeg->iRowid;
  4855         -  if( pIter->pIndex->pConfig->bOffsets 
         4857  +  if( eDetail==FTS5_DETAIL_FULL 
  4856   4858      && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  4857   4859     ){
  4858   4860       u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  4859   4861       if( pColset==0 || pIter->bFiltered ){
  4860   4862         *pn = pSeg->nPos;
  4861   4863         *pp = pPos;
  4862   4864       }else if( pColset->nCol==1 ){
................................................................................
  4867   4869         fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
  4868   4870         *pp = pIter->poslist.p;
  4869   4871         *pn = pIter->poslist.n;
  4870   4872       }
  4871   4873     }else{
  4872   4874       fts5BufferZero(&pIter->poslist);
  4873   4875       fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  4874         -    *pp = pIter->poslist.p;
         4876  +    if( eDetail==FTS5_DETAIL_FULL ){
         4877  +      *pp = pIter->poslist.p;
         4878  +    }
  4875   4879       *pn = pIter->poslist.n;
  4876   4880     }
  4877   4881     return fts5IndexReturn(pIter->pIndex);
  4878   4882   }
  4879   4883   
  4880   4884   /*
  4881   4885   ** This function is similar to sqlite3Fts5IterPoslist(), except that it
................................................................................
  5058   5062     const char *z,                  /* Index key to query for */
  5059   5063     int n,                          /* Size of index key in bytes */
  5060   5064     int flags,                      /* Flags for Fts5IndexQuery */
  5061   5065     u64 *pCksum                     /* IN/OUT: Checksum value */
  5062   5066   ){
  5063   5067     u64 cksum = *pCksum;
  5064   5068     Fts5IndexIter *pIdxIter = 0;
         5069  +  Fts5Buffer buf = {0, 0, 0};
  5065   5070     int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
  5066   5071   
  5067   5072     while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  5068         -    i64 dummy;
  5069         -    const u8 *pPos;
  5070         -    int nPos;
  5071   5073       i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
  5072         -    rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy);
         5074  +    rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
  5073   5075       if( rc==SQLITE_OK ){
  5074   5076         Fts5PoslistReader sReader;
  5075         -      for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader);
         5077  +      for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
  5076   5078             sReader.bEof==0;
  5077   5079             sqlite3Fts5PoslistReaderNext(&sReader)
  5078   5080         ){
  5079   5081           int iCol = FTS5_POS2COLUMN(sReader.iPos);
  5080   5082           int iOff = FTS5_POS2OFFSET(sReader.iPos);
  5081   5083           cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5082   5084         }
  5083   5085         rc = sqlite3Fts5IterNext(pIdxIter);
  5084   5086       }
  5085   5087     }
  5086   5088     sqlite3Fts5IterClose(pIdxIter);
         5089  +  fts5BufferFree(&buf);
  5087   5090   
  5088   5091     *pCksum = cksum;
  5089   5092     return rc;
  5090   5093   }
  5091   5094   
  5092   5095   
  5093   5096   /*

Changes to ext/fts5/fts5_main.c.

   222    222   */
   223    223   #define FTS5CSR_REQUIRE_CONTENT   0x01
   224    224   #define FTS5CSR_REQUIRE_DOCSIZE   0x02
   225    225   #define FTS5CSR_REQUIRE_INST      0x04
   226    226   #define FTS5CSR_EOF               0x08
   227    227   #define FTS5CSR_FREE_ZRANK        0x10
   228    228   #define FTS5CSR_REQUIRE_RESEEK    0x20
          229  +#define FTS5CSR_REQUIRE_POSLIST   0x40
   229    230   
   230    231   #define BitFlagAllTest(x,y) (((x) & (y))==(y))
   231    232   #define BitFlagTest(x,y)    (((x) & (y))!=0)
   232    233   
   233    234   
   234    235   /*
   235    236   ** Macros to Set(), Clear() and Test() cursor flags.
................................................................................
   309    310     return pTab->pConfig->eContent==FTS5_CONTENT_NONE;
   310    311   }
   311    312   
   312    313   /*
   313    314   ** Return true if pTab is an offsetless table.
   314    315   */
   315    316   static int fts5IsOffsetless(Fts5Table *pTab){
   316         -  return pTab->pConfig->bOffsets==0;
          317  +  return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL;
   317    318   }
   318    319   
   319    320   /*
   320    321   ** Delete a virtual table handle allocated by fts5InitVtab(). 
   321    322   */
   322    323   static void fts5FreeVtab(Fts5Table *pTab){
   323    324     if( pTab ){
................................................................................
   642    643   ** specific to the previous row stored by the cursor object.
   643    644   */
   644    645   static void fts5CsrNewrow(Fts5Cursor *pCsr){
   645    646     CsrFlagSet(pCsr, 
   646    647         FTS5CSR_REQUIRE_CONTENT 
   647    648       | FTS5CSR_REQUIRE_DOCSIZE 
   648    649       | FTS5CSR_REQUIRE_INST 
          650  +    | FTS5CSR_REQUIRE_POSLIST 
   649    651     );
   650    652   }
   651    653   
   652    654   static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
   653    655     Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
   654    656     Fts5Auxdata *pData;
   655    657     Fts5Auxdata *pNext;
................................................................................
  1599   1601     int rc;
  1600   1602     Fts5Table *pTab = (Fts5Table*)pVtab;
  1601   1603     fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0);
  1602   1604     rc = sqlite3Fts5StorageRollback(pTab->pStorage);
  1603   1605     return rc;
  1604   1606   }
  1605   1607   
         1608  +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**);
         1609  +
  1606   1610   static void *fts5ApiUserData(Fts5Context *pCtx){
  1607   1611     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1608   1612     return pCsr->pAux->pUserData;
  1609   1613   }
  1610   1614   
  1611   1615   static int fts5ApiColumnCount(Fts5Context *pCtx){
  1612   1616     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
................................................................................
  1647   1651     return sqlite3Fts5ExprPhraseCount(pCsr->pExpr);
  1648   1652   }
  1649   1653   
  1650   1654   static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
  1651   1655     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1652   1656     return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
  1653   1657   }
         1658  +
         1659  +static int fts5ApiColumnText(
         1660  +  Fts5Context *pCtx, 
         1661  +  int iCol, 
         1662  +  const char **pz, 
         1663  +  int *pn
         1664  +){
         1665  +  int rc = SQLITE_OK;
         1666  +  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
         1667  +  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
         1668  +    *pz = 0;
         1669  +    *pn = 0;
         1670  +  }else{
         1671  +    rc = fts5SeekCursor(pCsr, 0);
         1672  +    if( rc==SQLITE_OK ){
         1673  +      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
         1674  +      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
         1675  +    }
         1676  +  }
         1677  +  return rc;
         1678  +}
  1654   1679   
  1655   1680   static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
  1656   1681     int n;
         1682  +  int rc = SQLITE_OK;
         1683  +
         1684  +  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
         1685  +    Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
         1686  +    if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
         1687  +      Fts5PoslistWriter *aWriter;
         1688  +      int i;
         1689  +assert( pCsr->pSorter==0 );  /* fixme */
         1690  +      aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr);
         1691  +      if( aWriter==0 ) rc = SQLITE_NOMEM;
         1692  +      for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
         1693  +        int n; const char *z;
         1694  +        rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
         1695  +        if( rc==SQLITE_OK ){
         1696  +          rc = sqlite3Fts5ExprPopulatePoslists(
         1697  +              pConfig, pCsr->pExpr, aWriter, i, z, n
         1698  +          );
         1699  +        }
         1700  +      }
         1701  +      sqlite3_free(aWriter);
         1702  +    }
         1703  +    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
         1704  +  }
         1705  +
  1657   1706     if( pCsr->pSorter ){
  1658   1707       Fts5Sorter *pSorter = pCsr->pSorter;
  1659   1708       int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
  1660   1709       n = pSorter->aIdx[iPhrase] - i1;
  1661   1710       *pa = &pSorter->aPoslist[i1];
  1662   1711     }else{
  1663   1712       n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
................................................................................
  1752   1801     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1753   1802     int rc = SQLITE_OK;
  1754   1803     if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
  1755   1804      || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
  1756   1805     ){
  1757   1806       if( iIdx<0 || iIdx>=pCsr->nInstCount ){
  1758   1807         rc = SQLITE_RANGE;
         1808  +#if 0
  1759   1809       }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
  1760   1810         *piPhrase = pCsr->aInst[iIdx*3];
  1761   1811         *piCol = pCsr->aInst[iIdx*3 + 2];
  1762   1812         *piOff = -1;
         1813  +#endif
  1763   1814       }else{
  1764   1815         *piPhrase = pCsr->aInst[iIdx*3];
  1765   1816         *piCol = pCsr->aInst[iIdx*3 + 1];
  1766   1817         *piOff = pCsr->aInst[iIdx*3 + 2];
  1767   1818       }
  1768   1819     }
  1769   1820     return rc;
  1770   1821   }
  1771   1822   
  1772   1823   static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
  1773   1824     return fts5CursorRowid((Fts5Cursor*)pCtx);
  1774   1825   }
  1775   1826   
  1776         -static int fts5ApiColumnText(
  1777         -  Fts5Context *pCtx, 
  1778         -  int iCol, 
  1779         -  const char **pz, 
  1780         -  int *pn
  1781         -){
  1782         -  int rc = SQLITE_OK;
  1783         -  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1784         -  if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
  1785         -    *pz = 0;
  1786         -    *pn = 0;
  1787         -  }else{
  1788         -    rc = fts5SeekCursor(pCsr, 0);
  1789         -    if( rc==SQLITE_OK ){
  1790         -      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
  1791         -      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
  1792         -    }
  1793         -  }
  1794         -  return rc;
  1795         -}
  1796         -
  1797   1827   static int fts5ColumnSizeCb(
  1798   1828     void *pContext,                 /* Pointer to int */
  1799   1829     int tflags,
  1800   1830     const char *pToken,             /* Buffer containing token */
  1801   1831     int nToken,                     /* Size of token in bytes */
  1802   1832     int iStart,                     /* Start offset of token */
  1803   1833     int iEnd                        /* End offset of token */
................................................................................
  1921   1951     Fts5Context *pCtx, 
  1922   1952     Fts5PhraseIter *pIter, 
  1923   1953     int *piCol, int *piOff
  1924   1954   ){
  1925   1955     if( pIter->a>=pIter->b ){
  1926   1956       *piCol = -1;
  1927   1957       *piOff = -1;
         1958  +#if 0
  1928   1959     }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){
  1929   1960       int iVal;
  1930   1961       pIter->a += fts5GetVarint32(pIter->a, iVal);
  1931   1962       *piCol += (iVal-2);
  1932   1963       *piOff = -1;
         1964  +#endif
  1933   1965     }else{
  1934   1966       int iVal;
  1935   1967       pIter->a += fts5GetVarint32(pIter->a, iVal);
  1936   1968       if( iVal==1 ){
  1937   1969         pIter->a += fts5GetVarint32(pIter->a, iVal);
  1938   1970         *piCol = iVal;
  1939   1971         *piOff = 0;
................................................................................
  1977   2009     fts5ApiColumnSize,
  1978   2010     fts5ApiQueryPhrase,
  1979   2011     fts5ApiSetAuxdata,
  1980   2012     fts5ApiGetAuxdata,
  1981   2013     fts5ApiPhraseFirst,
  1982   2014     fts5ApiPhraseNext,
  1983   2015   };
  1984         -
  1985   2016   
  1986   2017   /*
  1987   2018   ** Implementation of API function xQueryPhrase().
  1988   2019   */
  1989   2020   static int fts5ApiQueryPhrase(
  1990   2021     Fts5Context *pCtx, 
  1991   2022     int iPhrase, 

Changes to ext/fts5/fts5_storage.c.

   912    912         if( pConfig->bColumnsize ){
   913    913           rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
   914    914         }
   915    915         for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
   916    916           if( pConfig->abUnindexed[i] ) continue;
   917    917           ctx.iCol = i;
   918    918           ctx.szCol = 0;
   919         -        if( pConfig->bOffsets==0 ){
          919  +        if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
   920    920             rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
   921    921           }
   922    922           if( rc==SQLITE_OK ){
   923    923             rc = sqlite3Fts5Tokenize(pConfig, 
   924    924                 FTS5_TOKENIZE_DOCUMENT,
   925    925                 (const char*)sqlite3_column_text(pScan, i+1),
   926    926                 sqlite3_column_bytes(pScan, i+1),

Changes to ext/fts5/test/fts5_common.tcl.

   235    235         
   236    236         for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
   237    237           set p [lindex $phraselist $iPhrase]
   238    238           set nPm1 [expr {[llength $p] - 1}]
   239    239           set iFirst [expr $iFL - $O(-near) - [llength $p]]
   240    240   
   241    241           for {set i $iFirst} {$i <= $iFL} {incr i} {
   242         -          if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
          242  +          set lCand [lrange $col $i [expr $i+$nPm1]]
          243  +
          244  +          set bMatch 1
          245  +          foreach tok $p term $lCand {
          246  +            if {[string match $tok $term]==0} {
          247  +              #puts "$tok $term failed"
          248  +              set bMatch 0
          249  +            }
          250  +          }
          251  +          if {$bMatch} { 
          252  +            #puts "match at $i"
          253  +            lappend B($iPhrase) $i 
          254  +          }
          255  +
          256  +          #if {$lCand == $p} { lappend B($iPhrase) $i }
   243    257           }
   244    258           if {[llength $B($iPhrase)] == 0} break
   245    259         }
   246    260   
   247    261         if {$iPhrase==$nPhrase} {
   248    262           for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
   249    263             set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)]
................................................................................
   261    275         foreach a $A($iCol,$iPhrase) {
   262    276           lappend res "$counter.$iCol.$a"
   263    277         }
   264    278       }
   265    279       incr counter
   266    280     }
   267    281   
   268         -  #puts $res
          282  +  #puts "$aCol -> $res"
   269    283     sort_poslist $res
   270    284   }
   271    285   
   272    286   #-------------------------------------------------------------------------
   273    287   # Usage:
   274    288   #
   275    289   #   sort_poslist LIST

Changes to ext/fts5/test/fts5ac.test.

   126    126   }
   127    127   
   128    128   # Argument $expr is an FTS5 match expression designed to be executed against
   129    129   # an FTS5 table with the following schema:
   130    130   # 
   131    131   #   CREATE VIRTUAL TABLE xy USING fts5(x, y);
   132    132   #
   133         -# Assuming the table contains the same records as stored int the global 
          133  +# Assuming the table contains the same records as stored in the global 
   134    134   # $::data array (see above), this function returns a list containing one
   135    135   # element for each match in the dataset. The elements are themselves lists
   136    136   # formatted as follows:
   137    137   #
   138    138   #   <rowid> {<phrase 0 matches> <phrase 1 matches>...}
   139    139   #
   140    140   # where each <phrase X matches> element is a list of phrase matches in the
................................................................................
   171    171     return [concat {*}$res]
   172    172   }
   173    173   
   174    174   #
   175    175   # End of test code
   176    176   #-------------------------------------------------------------------------
   177    177   
   178         -proc fts5_test_poslist {cmd} {
   179         -  set res [list]
   180         -  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
   181         -    lappend res [string map {{ } .} [$cmd xInst $i]]
   182         -  }
   183         -  set res
   184         -}
   185         -
   186    178   
   187    179   foreach {tn2 sql} {
   188    180     1  {}
   189    181     2  {BEGIN}
   190    182   } {
   191    183     reset_db
   192         -  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
          184  +  fts5_aux_test_functions db
   193    185   
   194         -  do_execsql_test 1.0 {
          186  +  do_execsql_test 1.$tn2.0 {
   195    187       CREATE VIRTUAL TABLE xx USING fts5(x,y);
   196    188       INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
   197    189     }
   198    190   
   199    191     execsql $sql
   200    192   
   201         -  do_test $tn2.1.1 {
          193  +  do_test 1.$tn2.1.1 {
   202    194       foreach {id x y} $data {
   203    195         execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
   204    196       }
   205    197       execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
   206    198     } {}
   207    199   
   208    200   
................................................................................
   220    212       8 "c"
   221    213       9 "no"
   222    214       10 "L O O L V V K"
   223    215     } {
   224    216       set expr "\"$phrase\""
   225    217       set res [matchdata 1 $expr]
   226    218   
   227         -    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
          219  +    do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { 
   228    220         SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   229    221       } $res
   230    222     }
   231    223   
   232    224     #-------------------------------------------------------------------------
   233    225     # Test some AND and OR queries.
   234    226     #
................................................................................
   242    234       2.2 "a+b OR c"
   243    235       2.3 "d+c OR u"
   244    236       2.4 "d+c OR u+d"
   245    237   
   246    238       3.1 { a AND b AND c }
   247    239     } {
   248    240       set res [matchdata 1 $expr]
   249         -    do_execsql_test $tn2.2.$tn.[llength $res] { 
          241  +    do_execsql_test 1.$tn2.2.$tn.[llength $res] { 
   250    242         SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   251    243       } $res
   252    244     }
   253    245   
   254    246     #-------------------------------------------------------------------------
   255    247     # Queries on a specific column.
   256    248     #
................................................................................
   271    263   
   272    264       4.1 {{"x" "y"}:a}
   273    265       4.2 {{"y" x}:a}
   274    266       4.3 {{x "x"}:b}
   275    267       4.4 {{"y" y}:b}
   276    268     } {
   277    269       set res [matchdata 1 $expr]
   278         -    do_execsql_test $tn2.3.$tn.[llength $res] { 
          270  +    do_execsql_test 1.$tn2.3.$tn.[llength $res] { 
   279    271         SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   280    272       } $res
   281    273     }
   282    274   
   283    275     #-------------------------------------------------------------------------
   284    276     # Some NEAR queries.
   285    277     #
................................................................................
   292    284       5 { NEAR(r c, 0) }
   293    285       6 { NEAR(a b c) }
   294    286       7 { NEAR(a b c, 8) }
   295    287       8  { x : NEAR(r c) }
   296    288       9  { y : NEAR(r c) }
   297    289     } {
   298    290       set res [matchdata 1 $expr]
   299         -    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
          291  +    do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { 
   300    292         SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
   301    293       } $res
   302    294     }
   303    295   
   304    296     do_test $tn2.4.1  { nearset {{a b c}} -- a } {0.0.0}
   305    297     do_test $tn2.4.2  { nearset {{a b c}} -- c } {0.0.2}
   306    298   
   307    299     foreach {tn expr tclexpr} {
   308    300       1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
   309    301     } {
   310         -    do_execsql_test $tn2.5.$tn {
          302  +    do_execsql_test 1.$tn2.5.$tn {
   311    303         SELECT fts5_expr_tcl($expr, 'N $x')
   312    304       } [list $tclexpr]
   313    305     }
   314    306   
   315    307     #-------------------------------------------------------------------------
   316    308     #
   317         -  do_execsql_test $tn2.6.integrity {
          309  +  do_execsql_test 1.$tn2.6.integrity {
   318    310       INSERT INTO xx(xx) VALUES('integrity-check');
   319    311     }
   320    312     #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
   321    313     foreach {bAsc sql} {
   322    314       1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
   323    315       0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
   324    316     } {
................................................................................
   342    334         15 { a OR b AND c }
   343    335         16 { c AND b OR a }
   344    336         17 { c AND (b OR a) }
   345    337         18 { c NOT (b OR a) }
   346    338         19 { c NOT b OR a AND d }
   347    339       } {
   348    340         set res [matchdata 0 $expr $bAsc]
   349         -      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
          341  +      do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res
   350    342       }
   351    343     }
   352    344   }
   353    345   
   354         -do_execsql_test 3.1 {
          346  +do_execsql_test 2.1 {
   355    347     SELECT fts5_expr_tcl('a AND b');
   356    348   } {{AND [nearset -- {a}] [nearset -- {b}]}}
          349  +
          350  +# Some tests for detail=col tables and detail=none.
          351  +#
          352  +foreach {tn2 sql} {
          353  +  1  {
          354  +    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
          355  +  }
          356  +  2  {
          357  +    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col);
          358  +    BEGIN;
          359  +  }
          360  +  3  {
          361  +    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
          362  +    BEGIN;
          363  +  }
          364  +  4  {
          365  +    CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none);
          366  +  }
          367  +} {
          368  +  reset_db
          369  +  fts5_aux_test_functions db
          370  +
          371  +  execsql $sql
          372  +
          373  +  do_execsql_test 3.$tn2.0 {
          374  +    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
          375  +  }
          376  +
          377  +
          378  +  do_test 3.$tn2.1.1 {
          379  +    foreach {id x y} $data {
          380  +      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
          381  +    }
          382  +    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
          383  +  } {}
          384  +
          385  +  foreach {tn q} {
          386  +    1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L"
          387  +    11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d"
          388  +  } {
          389  +    set res [matchdata 1 $q]
          390  +
          391  +    do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { 
          392  +      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q
          393  +    } $res
          394  +  }
          395  +
          396  +}
   357    397   
   358    398   finish_test
   359    399   

Changes to ext/fts5/test/fts5ad.test.

    71     71     5 {
    72     72       CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
    73     73       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    74     74       BEGIN;
    75     75     }
    76     76   
    77     77     6 {
    78         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0);
           78  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col);
    79     79       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    80     80     }
    81     81   
    82     82     7 {
    83         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
           83  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    84     84       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    85     85     }
    86     86   
    87     87     8 {
    88         -    CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5");
           88  +    CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5");
    89     89       INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
    90     90       BEGIN;
    91     91     }
    92     92   
    93     93   } {
    94     94   
    95     95     do_test $T.1 { 

Name change from ext/fts5/test/fts5offsets.test to ext/fts5/test/fts5detail.test.

     9      9   #
    10     10   #*************************************************************************
    11     11   # This file implements regression tests for SQLite library.  The
    12     12   # focus of this script is testing the FTS5 module.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16         -set testprefix fts5offsets
           16  +set testprefix fts5detail
    17     17   
    18     18   # If SQLITE_ENABLE_FTS5 is not defined, omit this file.
    19     19   ifcapable !fts5 {
    20     20     finish_test
    21     21     return
    22     22   }
           23  +
           24  +fts5_aux_test_functions db
    23     25   
    24     26   #--------------------------------------------------------------------------
    25     27   # Simple tests.
    26     28   #
    27     29   do_execsql_test 1.0 {
    28         -  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0);
           30  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
    29     31     INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
    30     32     INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
    31     33     INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
    32     34     INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
    33     35     INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
    34     36     INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
    35     37     INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
................................................................................
    54     56     do_execsql_test 1.2.$tn.2 {
    55     57       SELECT rowid FROM t1($match || '*');
    56     58     } $res
    57     59   }
    58     60   
    59     61   do_catchsql_test 1.3.1 {
    60     62     SELECT rowid FROM t1('h + d');
    61         -} {1 {fts5: phrase queries are not supported (offsets=0)}}
           63  +} {1 {fts5: phrase queries are not supported (detail!=full)}}
    62     64   
    63     65   do_catchsql_test 1.3.2 {
    64     66     SELECT rowid FROM t1('NEAR(h d)');
    65         -} {1 {fts5: NEAR queries are not supported (offsets=0)}}
           67  +} {1 {fts5: NEAR queries are not supported (detail!=full)}}
           68  +
    66     69   
    67     70   #-------------------------------------------------------------------------
    68         -# integrity-check with both offsets= and prefix= options.
           71  +# integrity-check with both detail= and prefix= options.
    69     72   #
    70     73   do_execsql_test 2.0 {
    71         -  CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1");
           74  +  CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
    72     75     INSERT INTO t2(a) VALUES('aa ab');
    73     76   }
    74     77   
    75     78   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}
    76     79   
    77     80   do_execsql_test 2.1 {
    78     81     INSERT INTO t2(t2) VALUES('integrity-check');
    79     82   }
           83  +
           84  +do_execsql_test 2.2 {
           85  +  SELECT fts5_test_poslist(t2) FROM t2('aa');
           86  +} {0.0.0}
           87  +
           88  +set ::pc 0
           89  +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
           90  +#exit
    80     91   
    81     92   #-------------------------------------------------------------------------
    82     93   # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
    83         -# work with offsets=0 tables.
           94  +# work with detail=col tables.
    84     95   #
    85     96   set data {
    86     97     1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
    87     98     2  {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
    88     99     3  {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
    89    100     4  {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
    90    101     5  {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}
................................................................................
   120    131     35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
   121    132     36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
   122    133     37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
   123    134     38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
   124    135     39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
   125    136     40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
   126    137   }
          138  +
          139  +set data {
          140  +  1  {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
          141  +}
          142  +
          143  +proc matchdata {expr {bAsc 1}} {
          144  +
          145  +  set tclexpr [db one {
          146  +    SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
          147  +  }]
          148  +  set res [list]
          149  +
          150  +  #puts "$expr -> $tclexpr"
          151  +  foreach {id x y z} $::data {
          152  +    set cols [list $x $y $z]
          153  +    set ::pc 0
          154  +    #set hits [lsort -command instcompare [eval $tclexpr]]
          155  +    set hits [eval $tclexpr]
          156  +    if {[llength $hits]>0} {
          157  +      lappend res [list $id $hits]
          158  +    }
          159  +  }
          160  +
          161  +  if {$bAsc} {
          162  +    set res [lsort -integer -increasing -index 0 $res]
          163  +  } else {
          164  +    set res [lsort -integer -decreasing -index 0 $res]
          165  +  }
          166  +
          167  +  return [concat {*}$res]
          168  +}
          169  +
   127    170   foreach {tn tbl} {
   128         -  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) }
          171  +  1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
   129    172   } {
          173  +#break
   130    174     reset_db
   131    175     fts5_aux_test_functions db
   132    176     execsql $tbl
   133    177     foreach {id x y z} $data {
   134    178       execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
   135    179     }
   136    180     foreach {tn2 expr} {
   137    181       1 aaa    2 ccc    3 bab    4 aac
   138    182       5 aa*    6 cc*    7 ba*    8 aa*
   139    183       9 a*     10 b*   11 c*
   140    184     } {
   141    185   
   142         -    set res [list]
   143         -    foreach {id x y z} $data {
   144         -      if {[lsearch [concat $x $y $z] $expr]>=0} {
   145         -        lappend res $id
   146         -        set inst [list]
   147         -        if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 }
   148         -        if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 }
   149         -        if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 }
   150         -        lappend res $inst
   151         -      }
   152         -    }
          186  +    set res [matchdata $expr]
   153    187   
   154    188       do_execsql_test 3.$tn.$tn2.1 {
   155    189         SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
   156    190       } $res
   157    191   
   158    192       do_execsql_test 3.$tn.$tn2.2 {
   159    193         SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
   160    194       } $res
   161    195     }
          196  +}
   162    197   
          198  +#-------------------------------------------------------------------------
          199  +# Simple tests for detail=none tables.
          200  +#
          201  +breakpoint
          202  +do_execsql_test 4.0 {
          203  +  CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
          204  +  INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
          205  +  INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
   163    206   }
   164    207   
          208  +do_catchsql_test 4.1 {
          209  +  SELECT * FROM t4('a:a')
          210  +} {1 {fts5: column queries are not supported (detail=none)}}
          211  +
   165    212   finish_test
   166    213