Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5-offsets |
| Files: | files | file ages | folders |
| SHA1: | 228b4d10e38f7d70e1b008c3c9b4a1ae |
| User & Date: | dan 2015-12-28 19:55:00 |
Context
|
2015-12-29
| ||
| 19:35 | Add the xPhraseFirstColumn() and xPhraseNextColumn() API functions to fts5. For iterating through the set of columns that contain intances of a phrase. check-in: 8c30605b user: dan tags: fts5-offsets | |
|
2015-12-28
| ||
| 19:55 | Change the name of the offsets=0 option to "detail=column". Have the xInst, xPhraseFirst and other API functions work by parsing the original text for detail=column tables. check-in: 228b4d10 user: dan tags: fts5-offsets | |
|
2015-12-22
| ||
| 18:54 | Updates so that fts5 API functions xInst, xPhraseFirst and xPhraseNext work with the offsets=0 option. check-in: 69bffc16 user: dan tags: fts5-offsets | |
Changes
Changes to ext/fts5/fts5Int.h.
147 147 u8 *abUnindexed; /* True for unindexed columns */ 148 148 int nPrefix; /* Number of prefix indexes */ 149 149 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ 150 150 int eContent; /* An FTS5_CONTENT value */ 151 151 char *zContent; /* content table */ 152 152 char *zContentRowid; /* "content_rowid=" option value */ 153 153 int bColumnsize; /* "columnsize=" option value (dflt==1) */ 154 - int bOffsets; /* "offsets=" option value (dflt==1) */ 154 + int eDetail; /* FTS5_DETAIL_XXX value */ 155 155 char *zContentExprlist; 156 156 Fts5Tokenizer *pTok; 157 157 fts5_tokenizer *pTokApi; 158 158 159 159 /* Values loaded from the %_config table */ 160 160 int iCookie; /* Incremented when %_config is modified */ 161 161 int pgsz; /* Approximate page size used in %_data */ ................................................................................ 176 176 /* Current expected value of %_config table 'version' field */ 177 177 #define FTS5_CURRENT_VERSION 4 178 178 179 179 #define FTS5_CONTENT_NORMAL 0 180 180 #define FTS5_CONTENT_NONE 1 181 181 #define FTS5_CONTENT_EXTERNAL 2 182 182 183 +#define FTS5_DETAIL_FULL 0 184 +#define FTS5_DETAIL_NONE 1 185 +#define FTS5_DETAIL_COLUMNS 2 183 186 184 187 185 188 186 189 int sqlite3Fts5ConfigParse( 187 190 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** 188 191 ); 189 192 void sqlite3Fts5ConfigFree(Fts5Config*); ................................................................................ 632 635 633 636 /* Called during startup to register a UDF with SQLite */ 634 637 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); 635 638 636 639 int sqlite3Fts5ExprPhraseCount(Fts5Expr*); 637 640 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); 638 641 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **); 642 + 643 +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr*); 644 +int sqlite3Fts5ExprPopulatePoslists( 645 + Fts5Config*, Fts5Expr*, Fts5PoslistWriter*, int, const char*, int 646 +); 639 647 640 648 int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**); 641 649 642 650 /******************************************* 643 651 ** The fts5_expr.c API above this point is used by the other hand-written 644 652 ** C code in this module. The interfaces below this point are called by 645 653 ** the parser code in fts5parse.y. */
Changes to ext/fts5/fts5_config.c.
190 190 assert( 0==fts5_iswhitespace(z[0]) ); 191 191 quote = z[0]; 192 192 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ 193 193 fts5Dequote(z); 194 194 } 195 195 } 196 196 197 + 198 +struct Fts5Enum { 199 + const char *zName; 200 + int eVal; 201 +}; 202 +typedef struct Fts5Enum Fts5Enum; 203 + 204 +static int fts5ConfigSetEnum( 205 + const Fts5Enum *aEnum, 206 + const char *zEnum, 207 + int *peVal 208 +){ 209 + int nEnum = strlen(zEnum); 210 + int i; 211 + int iVal = -1; 212 + 213 + for(i=0; aEnum[i].zName; i++){ 214 + if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){ 215 + if( iVal>=0 ) return SQLITE_ERROR; 216 + iVal = aEnum[i].eVal; 217 + } 218 + } 219 + 220 + *peVal = iVal; 221 + return iVal<0 ? SQLITE_ERROR : SQLITE_OK; 222 +} 223 + 197 224 /* 198 225 ** Parse a "special" CREATE VIRTUAL TABLE directive and update 199 226 ** configuration object pConfig as appropriate. 200 227 ** 201 228 ** If successful, object pConfig is updated and SQLITE_OK returned. If 202 229 ** an error occurs, an SQLite error code is returned and an error message 203 230 ** may be left in *pzErr. It is the responsibility of the caller to ................................................................................ 340 367 rc = SQLITE_ERROR; 341 368 }else{ 342 369 pConfig->bColumnsize = (zArg[0]=='1'); 343 370 } 344 371 return rc; 345 372 } 346 373 347 - if( sqlite3_strnicmp("offsets", zCmd, nCmd)==0 ){ 348 - if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ 349 - *pzErr = sqlite3_mprintf("malformed offsets=... directive"); 350 - rc = SQLITE_ERROR; 351 - }else{ 352 - pConfig->bOffsets = (zArg[0]=='1'); 374 + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ 375 + const Fts5Enum aDetail[] = { 376 + { "none", FTS5_DETAIL_NONE }, 377 + { "full", FTS5_DETAIL_FULL }, 378 + { "columns", FTS5_DETAIL_COLUMNS }, 379 + { 0, 0 } 380 + }; 381 + 382 + if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){ 383 + *pzErr = sqlite3_mprintf("malformed detail=... directive"); 353 384 } 354 385 return rc; 355 386 } 356 387 357 388 *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); 358 389 return SQLITE_ERROR; 359 390 } ................................................................................ 505 536 506 537 nByte = nArg * (sizeof(char*) + sizeof(u8)); 507 538 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); 508 539 pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; 509 540 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); 510 541 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); 511 542 pRet->bColumnsize = 1; 512 - pRet->bOffsets = 1; 543 + pRet->eDetail = FTS5_DETAIL_FULL; 513 544 #ifdef SQLITE_DEBUG 514 545 pRet->bPrefixIndex = 1; 515 546 #endif 516 547 if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ 517 548 *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); 518 549 rc = SQLITE_ERROR; 519 550 }
Changes to ext/fts5/fts5_expr.c.
1740 1740 } 1741 1741 1742 1742 void sqlite3Fts5ParseSetColset( 1743 1743 Fts5Parse *pParse, 1744 1744 Fts5ExprNearset *pNear, 1745 1745 Fts5Colset *pColset 1746 1746 ){ 1747 + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ 1748 + pParse->rc = SQLITE_ERROR; 1749 + pParse->zErr = sqlite3_mprintf( 1750 + "fts5: column queries are not supported (detail=none)" 1751 + ); 1752 + sqlite3_free(pColset); 1753 + return; 1754 + } 1755 + 1747 1756 if( pNear ){ 1748 1757 pNear->pColset = pColset; 1749 1758 }else{ 1750 1759 sqlite3_free(pColset); 1751 1760 } 1752 1761 } 1753 1762 ................................................................................ 1805 1814 for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){ 1806 1815 pNear->apPhrase[iPhrase]->pNode = pRet; 1807 1816 } 1808 1817 if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){ 1809 1818 if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){ 1810 1819 pRet->eType = FTS5_TERM; 1811 1820 } 1812 - }else if( pParse->pConfig->bOffsets==0 ){ 1821 + }else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ 1813 1822 assert( pParse->rc==SQLITE_OK ); 1814 1823 pParse->rc = SQLITE_ERROR; 1815 1824 assert( pParse->zErr==0 ); 1816 1825 pParse->zErr = sqlite3_mprintf( 1817 - "fts5: %s queries are not supported (offsets=0)", 1826 + "fts5: %s queries are not supported (detail!=full)", 1818 1827 pNear->nPhrase==1 ? "phrase": "NEAR" 1819 1828 ); 1820 1829 sqlite3_free(pRet); 1821 1830 pRet = 0; 1822 1831 } 1823 1832 }else{ 1824 1833 fts5ExprAddChildren(pRet, pLeft); ................................................................................ 1928 1937 for(i=0; i<pNear->nPhrase; i++){ 1929 1938 Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; 1930 1939 1931 1940 zRet = fts5PrintfAppend(zRet, " {"); 1932 1941 for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){ 1933 1942 char *zTerm = pPhrase->aTerm[iTerm].zTerm; 1934 1943 zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); 1944 + if( pPhrase->aTerm[iTerm].bPrefix ){ 1945 + zRet = fts5PrintfAppend(zRet, "*"); 1946 + } 1935 1947 } 1936 1948 1937 1949 if( zRet ) zRet = fts5PrintfAppend(zRet, "}"); 1938 1950 if( zRet==0 ) return 0; 1939 1951 } 1940 1952 1941 1953 }else{ ................................................................................ 2240 2252 nRet = pPhrase->poslist.n; 2241 2253 }else{ 2242 2254 *pa = 0; 2243 2255 nRet = 0; 2244 2256 } 2245 2257 return nRet; 2246 2258 } 2259 + 2260 +Fts5PoslistWriter *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr){ 2261 + int i; 2262 + Fts5PoslistWriter *pRet; 2263 + for(i=0; i<pExpr->nPhrase; i++){ 2264 + Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; 2265 + assert( pExpr->apExprPhrase[i]->nTerm==1 ); 2266 + pBuf->n = 0; 2267 + } 2268 + pRet = sqlite3_malloc(sizeof(Fts5PoslistWriter)*pExpr->nPhrase); 2269 + if( pRet ){ 2270 + memset(pRet, 0, sizeof(Fts5PoslistWriter)*pExpr->nPhrase); 2271 + } 2272 + return pRet; 2273 +} 2274 + 2275 +struct Fts5ExprCtx { 2276 + Fts5Expr *pExpr; 2277 + Fts5PoslistWriter *aWriter; 2278 + i64 iOff; 2279 +}; 2280 +typedef struct Fts5ExprCtx Fts5ExprCtx; 2281 + 2282 +static int fts5ExprPopulatePoslistsCb( 2283 + void *pCtx, /* Copy of 2nd argument to xTokenize() */ 2284 + int tflags, /* Mask of FTS5_TOKEN_* flags */ 2285 + const char *pToken, /* Pointer to buffer containing token */ 2286 + int nToken, /* Size of token in bytes */ 2287 + int iStart, /* Byte offset of token within input text */ 2288 + int iEnd /* Byte offset of end of token within input text */ 2289 +){ 2290 + Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx; 2291 + Fts5Expr *pExpr = p->pExpr; 2292 + int i; 2293 + 2294 + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; 2295 + for(i=0; i<pExpr->nPhrase; i++){ 2296 + Fts5ExprTerm *pTerm; 2297 + for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ 2298 + int nTerm = strlen(pTerm->zTerm); 2299 + if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix)) 2300 + && memcmp(pTerm->zTerm, pToken, nTerm)==0 2301 + ){ 2302 + int rc = sqlite3Fts5PoslistWriterAppend( 2303 + &pExpr->apExprPhrase[i]->poslist, &p->aWriter[i], p->iOff 2304 + ); 2305 + if( rc ) return rc; 2306 + break; 2307 + } 2308 + } 2309 + } 2310 + return SQLITE_OK; 2311 +} 2312 + 2313 +int sqlite3Fts5ExprPopulatePoslists( 2314 + Fts5Config *pConfig, 2315 + Fts5Expr *pExpr, 2316 + Fts5PoslistWriter *aWriter, 2317 + int iCol, 2318 + const char *z, int n 2319 +){ 2320 + Fts5ExprCtx sCtx; 2321 + sCtx.pExpr = pExpr; 2322 + sCtx.aWriter = aWriter; 2323 + sCtx.iOff = (((i64)iCol) << 32) - 1; 2324 + 2325 + return sqlite3Fts5Tokenize(pConfig, 2326 + FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb 2327 + ); 2328 +} 2329 +
Changes to ext/fts5/fts5_hash.c.
22 22 ** This file contains the implementation of an in-memory hash table used 23 23 ** to accumuluate "term -> doclist" content before it is flused to a level-0 24 24 ** segment. 25 25 */ 26 26 27 27 28 28 struct Fts5Hash { 29 - int bOffsets; /* Copy of Fts5Config.bOffsets */ 29 + int eDetail; /* Copy of Fts5Config.eDetail */ 30 30 int *pnByte; /* Pointer to bytes counter */ 31 31 int nEntry; /* Number of entries currently in hash */ 32 32 int nSlot; /* Size of aSlot[] array */ 33 33 Fts5HashEntry *pScan; /* Current ordered scan item */ 34 34 Fts5HashEntry **aSlot; /* Array of hash slots */ 35 35 }; 36 36 ................................................................................ 87 87 *ppNew = pNew = (Fts5Hash*)sqlite3_malloc(sizeof(Fts5Hash)); 88 88 if( pNew==0 ){ 89 89 rc = SQLITE_NOMEM; 90 90 }else{ 91 91 int nByte; 92 92 memset(pNew, 0, sizeof(Fts5Hash)); 93 93 pNew->pnByte = pnByte; 94 - pNew->bOffsets = pConfig->bOffsets; 94 + pNew->eDetail = pConfig->eDetail; 95 95 96 96 pNew->nSlot = 1024; 97 97 nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; 98 98 pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); 99 99 if( pNew->aSlot==0 ){ 100 100 sqlite3_free(pNew); 101 101 *ppNew = 0; ................................................................................ 212 212 char bByte, /* First byte of token */ 213 213 const char *pToken, int nToken /* Token to add or remove to or from index */ 214 214 ){ 215 215 unsigned int iHash; 216 216 Fts5HashEntry *p; 217 217 u8 *pPtr; 218 218 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ 219 - int bNew = pHash->bOffsets; /* If non-delete entry should be written */ 219 + int bNew; /* If non-delete entry should be written */ 220 + 221 + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); 220 222 221 223 /* Attempt to locate an existing hash entry */ 222 224 iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); 223 225 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ 224 226 if( p->zKey[0]==bByte 225 227 && memcmp(&p->zKey[1], pToken, nToken)==0 226 228 && p->zKey[nToken+1]==0 ................................................................................ 249 251 assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); 250 252 p->zKey[nToken+1] = '\0'; 251 253 p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; 252 254 p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); 253 255 p->iSzPoslist = p->nData; 254 256 p->nData += 1; 255 257 p->iRowid = iRowid; 256 - p->iCol = (pHash->bOffsets-1); 258 + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); 257 259 p->pHashNext = pHash->aSlot[iHash]; 258 260 pHash->aSlot[iHash] = p; 259 261 pHash->nEntry++; 260 262 nIncr += p->nData; 261 263 } 262 264 263 265 /* Check there is enough space to append a new entry. Worst case scenario ................................................................................ 286 288 /* If this is a new rowid, append the 4-byte size field for the previous 287 289 ** entry, and the new rowid for this entry. */ 288 290 if( iRowid!=p->iRowid ){ 289 291 fts5HashAddPoslistSize(p); 290 292 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); 291 293 p->iSzPoslist = p->nData; 292 294 p->nData += 1; 293 - p->iCol = (pHash->bOffsets-1); 295 + p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); 294 296 p->iPos = 0; 295 297 p->iRowid = iRowid; 296 298 bNew = 1; 297 299 } 298 300 299 301 if( iCol>=0 ){ 300 302 /* Append a new column value, if necessary */ 301 303 assert( iCol>=p->iCol ); 302 304 if( iCol!=p->iCol ){ 303 - if( pHash->bOffsets==0 ){ 304 - bNew = 1; 305 - p->iCol = iPos = iCol; 306 - }else{ 305 + if( pHash->eDetail==FTS5_DETAIL_FULL ){ 307 306 pPtr[p->nData++] = 0x01; 308 307 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol); 309 308 p->iCol = iCol; 310 309 p->iPos = 0; 310 + }else{ 311 + bNew = 1; 312 + p->iCol = iPos = iCol; 311 313 } 312 314 } 313 315 314 316 /* Append the new position offset, if necessary */ 315 317 if( bNew ){ 316 318 p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2); 317 319 p->iPos = iPos;
Changes to ext/fts5/fts5_index.c.
4105 4105 Fts5Colset *pColset, 4106 4106 Fts5Buffer *pBuf 4107 4107 ){ 4108 4108 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ 4109 4109 if( pColset==0 ){ 4110 4110 fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); 4111 4111 }else{ 4112 - if( p->pConfig->bOffsets==0 ){ 4113 - PoslistOffsetsCtx sCtx; 4114 - memset(&sCtx, 0, sizeof(sCtx)); 4115 - sCtx.pBuf = pBuf; 4116 - sCtx.pColset = pColset; 4117 - fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); 4118 - }else{ 4112 + if( p->pConfig->eDetail==FTS5_DETAIL_FULL ){ 4119 4113 PoslistCallbackCtx sCtx; 4120 4114 sCtx.pBuf = pBuf; 4121 4115 sCtx.pColset = pColset; 4122 4116 sCtx.eState = fts5IndexColsetTest(pColset, 0); 4123 4117 assert( sCtx.eState==0 || sCtx.eState==1 ); 4124 4118 fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); 4119 + }else{ 4120 + PoslistOffsetsCtx sCtx; 4121 + memset(&sCtx, 0, sizeof(sCtx)); 4122 + sCtx.pBuf = pBuf; 4123 + sCtx.pColset = pColset; 4124 + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistOffsetsCallback); 4125 4125 } 4126 4126 } 4127 4127 } 4128 4128 } 4129 4129 4130 4130 /* 4131 4131 ** IN/OUT parameter (*pa) points to a position list n bytes in size. If ................................................................................ 4188 4188 Fts5Buffer *pBuf 4189 4189 ){ 4190 4190 if( p->rc==SQLITE_OK ){ 4191 4191 Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; 4192 4192 assert( fts5MultiIterEof(p, pMulti)==0 ); 4193 4193 assert( pSeg->nPos>0 ); 4194 4194 if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ 4195 - if( p->pConfig->bOffsets 4195 + if( p->pConfig->eDetail==FTS5_DETAIL_FULL 4196 4196 && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 4197 4197 && (pColset==0 || pColset->nCol==1) 4198 4198 ){ 4199 4199 const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; 4200 4200 int nPos; 4201 4201 if( pColset ){ 4202 4202 nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); ................................................................................ 4846 4846 Fts5IndexIter *pIter, 4847 4847 Fts5Colset *pColset, /* Column filter (or NULL) */ 4848 4848 const u8 **pp, /* OUT: Pointer to position-list data */ 4849 4849 int *pn, /* OUT: Size of position-list in bytes */ 4850 4850 i64 *piRowid /* OUT: Current rowid */ 4851 4851 ){ 4852 4852 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; 4853 + int eDetail = pIter->pIndex->pConfig->eDetail; 4854 + 4853 4855 assert( pIter->pIndex->rc==SQLITE_OK ); 4854 4856 *piRowid = pSeg->iRowid; 4855 - if( pIter->pIndex->pConfig->bOffsets 4857 + if( eDetail==FTS5_DETAIL_FULL 4856 4858 && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 4857 4859 ){ 4858 4860 u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; 4859 4861 if( pColset==0 || pIter->bFiltered ){ 4860 4862 *pn = pSeg->nPos; 4861 4863 *pp = pPos; 4862 4864 }else if( pColset->nCol==1 ){ ................................................................................ 4867 4869 fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); 4868 4870 *pp = pIter->poslist.p; 4869 4871 *pn = pIter->poslist.n; 4870 4872 } 4871 4873 }else{ 4872 4874 fts5BufferZero(&pIter->poslist); 4873 4875 fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); 4874 - *pp = pIter->poslist.p; 4876 + if( eDetail==FTS5_DETAIL_FULL ){ 4877 + *pp = pIter->poslist.p; 4878 + } 4875 4879 *pn = pIter->poslist.n; 4876 4880 } 4877 4881 return fts5IndexReturn(pIter->pIndex); 4878 4882 } 4879 4883 4880 4884 /* 4881 4885 ** This function is similar to sqlite3Fts5IterPoslist(), except that it ................................................................................ 5058 5062 const char *z, /* Index key to query for */ 5059 5063 int n, /* Size of index key in bytes */ 5060 5064 int flags, /* Flags for Fts5IndexQuery */ 5061 5065 u64 *pCksum /* IN/OUT: Checksum value */ 5062 5066 ){ 5063 5067 u64 cksum = *pCksum; 5064 5068 Fts5IndexIter *pIdxIter = 0; 5069 + Fts5Buffer buf = {0, 0, 0}; 5065 5070 int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); 5066 5071 5067 5072 while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ 5068 - i64 dummy; 5069 - const u8 *pPos; 5070 - int nPos; 5071 5073 i64 rowid = sqlite3Fts5IterRowid(pIdxIter); 5072 - rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); 5074 + rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf); 5073 5075 if( rc==SQLITE_OK ){ 5074 5076 Fts5PoslistReader sReader; 5075 - for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); 5077 + for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader); 5076 5078 sReader.bEof==0; 5077 5079 sqlite3Fts5PoslistReaderNext(&sReader) 5078 5080 ){ 5079 5081 int iCol = FTS5_POS2COLUMN(sReader.iPos); 5080 5082 int iOff = FTS5_POS2OFFSET(sReader.iPos); 5081 5083 cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); 5082 5084 } 5083 5085 rc = sqlite3Fts5IterNext(pIdxIter); 5084 5086 } 5085 5087 } 5086 5088 sqlite3Fts5IterClose(pIdxIter); 5089 + fts5BufferFree(&buf); 5087 5090 5088 5091 *pCksum = cksum; 5089 5092 return rc; 5090 5093 } 5091 5094 5092 5095 5093 5096 /*
Changes to ext/fts5/fts5_main.c.
222 222 */ 223 223 #define FTS5CSR_REQUIRE_CONTENT 0x01 224 224 #define FTS5CSR_REQUIRE_DOCSIZE 0x02 225 225 #define FTS5CSR_REQUIRE_INST 0x04 226 226 #define FTS5CSR_EOF 0x08 227 227 #define FTS5CSR_FREE_ZRANK 0x10 228 228 #define FTS5CSR_REQUIRE_RESEEK 0x20 229 +#define FTS5CSR_REQUIRE_POSLIST 0x40 229 230 230 231 #define BitFlagAllTest(x,y) (((x) & (y))==(y)) 231 232 #define BitFlagTest(x,y) (((x) & (y))!=0) 232 233 233 234 234 235 /* 235 236 ** Macros to Set(), Clear() and Test() cursor flags. ................................................................................ 309 310 return pTab->pConfig->eContent==FTS5_CONTENT_NONE; 310 311 } 311 312 312 313 /* 313 314 ** Return true if pTab is an offsetless table. 314 315 */ 315 316 static int fts5IsOffsetless(Fts5Table *pTab){ 316 - return pTab->pConfig->bOffsets==0; 317 + return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL; 317 318 } 318 319 319 320 /* 320 321 ** Delete a virtual table handle allocated by fts5InitVtab(). 321 322 */ 322 323 static void fts5FreeVtab(Fts5Table *pTab){ 323 324 if( pTab ){ ................................................................................ 642 643 ** specific to the previous row stored by the cursor object. 643 644 */ 644 645 static void fts5CsrNewrow(Fts5Cursor *pCsr){ 645 646 CsrFlagSet(pCsr, 646 647 FTS5CSR_REQUIRE_CONTENT 647 648 | FTS5CSR_REQUIRE_DOCSIZE 648 649 | FTS5CSR_REQUIRE_INST 650 + | FTS5CSR_REQUIRE_POSLIST 649 651 ); 650 652 } 651 653 652 654 static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ 653 655 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); 654 656 Fts5Auxdata *pData; 655 657 Fts5Auxdata *pNext; ................................................................................ 1599 1601 int rc; 1600 1602 Fts5Table *pTab = (Fts5Table*)pVtab; 1601 1603 fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); 1602 1604 rc = sqlite3Fts5StorageRollback(pTab->pStorage); 1603 1605 return rc; 1604 1606 } 1605 1607 1608 +static int fts5CsrPoslist(Fts5Cursor*, int, const u8**); 1609 + 1606 1610 static void *fts5ApiUserData(Fts5Context *pCtx){ 1607 1611 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1608 1612 return pCsr->pAux->pUserData; 1609 1613 } 1610 1614 1611 1615 static int fts5ApiColumnCount(Fts5Context *pCtx){ 1612 1616 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; ................................................................................ 1647 1651 return sqlite3Fts5ExprPhraseCount(pCsr->pExpr); 1648 1652 } 1649 1653 1650 1654 static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ 1651 1655 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1652 1656 return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); 1653 1657 } 1658 + 1659 +static int fts5ApiColumnText( 1660 + Fts5Context *pCtx, 1661 + int iCol, 1662 + const char **pz, 1663 + int *pn 1664 +){ 1665 + int rc = SQLITE_OK; 1666 + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1667 + if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ 1668 + *pz = 0; 1669 + *pn = 0; 1670 + }else{ 1671 + rc = fts5SeekCursor(pCsr, 0); 1672 + if( rc==SQLITE_OK ){ 1673 + *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1674 + *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1675 + } 1676 + } 1677 + return rc; 1678 +} 1654 1679 1655 1680 static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){ 1656 1681 int n; 1682 + int rc = SQLITE_OK; 1683 + 1684 + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ 1685 + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; 1686 + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ 1687 + Fts5PoslistWriter *aWriter; 1688 + int i; 1689 +assert( pCsr->pSorter==0 ); /* fixme */ 1690 + aWriter = sqlite3Fts5ExprClearPoslists(pCsr->pExpr); 1691 + if( aWriter==0 ) rc = SQLITE_NOMEM; 1692 + for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){ 1693 + int n; const char *z; 1694 + rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); 1695 + if( rc==SQLITE_OK ){ 1696 + rc = sqlite3Fts5ExprPopulatePoslists( 1697 + pConfig, pCsr->pExpr, aWriter, i, z, n 1698 + ); 1699 + } 1700 + } 1701 + sqlite3_free(aWriter); 1702 + } 1703 + CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); 1704 + } 1705 + 1657 1706 if( pCsr->pSorter ){ 1658 1707 Fts5Sorter *pSorter = pCsr->pSorter; 1659 1708 int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); 1660 1709 n = pSorter->aIdx[iPhrase] - i1; 1661 1710 *pa = &pSorter->aPoslist[i1]; 1662 1711 }else{ 1663 1712 n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); ................................................................................ 1752 1801 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1753 1802 int rc = SQLITE_OK; 1754 1803 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 1755 1804 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 1756 1805 ){ 1757 1806 if( iIdx<0 || iIdx>=pCsr->nInstCount ){ 1758 1807 rc = SQLITE_RANGE; 1808 +#if 0 1759 1809 }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ 1760 1810 *piPhrase = pCsr->aInst[iIdx*3]; 1761 1811 *piCol = pCsr->aInst[iIdx*3 + 2]; 1762 1812 *piOff = -1; 1813 +#endif 1763 1814 }else{ 1764 1815 *piPhrase = pCsr->aInst[iIdx*3]; 1765 1816 *piCol = pCsr->aInst[iIdx*3 + 1]; 1766 1817 *piOff = pCsr->aInst[iIdx*3 + 2]; 1767 1818 } 1768 1819 } 1769 1820 return rc; 1770 1821 } 1771 1822 1772 1823 static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){ 1773 1824 return fts5CursorRowid((Fts5Cursor*)pCtx); 1774 1825 } 1775 1826 1776 -static int fts5ApiColumnText( 1777 - Fts5Context *pCtx, 1778 - int iCol, 1779 - const char **pz, 1780 - int *pn 1781 -){ 1782 - int rc = SQLITE_OK; 1783 - Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1784 - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ 1785 - *pz = 0; 1786 - *pn = 0; 1787 - }else{ 1788 - rc = fts5SeekCursor(pCsr, 0); 1789 - if( rc==SQLITE_OK ){ 1790 - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1791 - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1792 - } 1793 - } 1794 - return rc; 1795 -} 1796 - 1797 1827 static int fts5ColumnSizeCb( 1798 1828 void *pContext, /* Pointer to int */ 1799 1829 int tflags, 1800 1830 const char *pToken, /* Buffer containing token */ 1801 1831 int nToken, /* Size of token in bytes */ 1802 1832 int iStart, /* Start offset of token */ 1803 1833 int iEnd /* End offset of token */ ................................................................................ 1921 1951 Fts5Context *pCtx, 1922 1952 Fts5PhraseIter *pIter, 1923 1953 int *piCol, int *piOff 1924 1954 ){ 1925 1955 if( pIter->a>=pIter->b ){ 1926 1956 *piCol = -1; 1927 1957 *piOff = -1; 1958 +#if 0 1928 1959 }else if( fts5IsOffsetless((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab)) ){ 1929 1960 int iVal; 1930 1961 pIter->a += fts5GetVarint32(pIter->a, iVal); 1931 1962 *piCol += (iVal-2); 1932 1963 *piOff = -1; 1964 +#endif 1933 1965 }else{ 1934 1966 int iVal; 1935 1967 pIter->a += fts5GetVarint32(pIter->a, iVal); 1936 1968 if( iVal==1 ){ 1937 1969 pIter->a += fts5GetVarint32(pIter->a, iVal); 1938 1970 *piCol = iVal; 1939 1971 *piOff = 0; ................................................................................ 1977 2009 fts5ApiColumnSize, 1978 2010 fts5ApiQueryPhrase, 1979 2011 fts5ApiSetAuxdata, 1980 2012 fts5ApiGetAuxdata, 1981 2013 fts5ApiPhraseFirst, 1982 2014 fts5ApiPhraseNext, 1983 2015 }; 1984 - 1985 2016 1986 2017 /* 1987 2018 ** Implementation of API function xQueryPhrase(). 1988 2019 */ 1989 2020 static int fts5ApiQueryPhrase( 1990 2021 Fts5Context *pCtx, 1991 2022 int iPhrase,
Changes to ext/fts5/fts5_storage.c.
912 912 if( pConfig->bColumnsize ){ 913 913 rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); 914 914 } 915 915 for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ 916 916 if( pConfig->abUnindexed[i] ) continue; 917 917 ctx.iCol = i; 918 918 ctx.szCol = 0; 919 - if( pConfig->bOffsets==0 ){ 919 + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ 920 920 rc = sqlite3Fts5TermsetNew(&ctx.pTermset); 921 921 } 922 922 if( rc==SQLITE_OK ){ 923 923 rc = sqlite3Fts5Tokenize(pConfig, 924 924 FTS5_TOKENIZE_DOCUMENT, 925 925 (const char*)sqlite3_column_text(pScan, i+1), 926 926 sqlite3_column_bytes(pScan, i+1),
Changes to ext/fts5/test/fts5_common.tcl.
235 235 236 236 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { 237 237 set p [lindex $phraselist $iPhrase] 238 238 set nPm1 [expr {[llength $p] - 1}] 239 239 set iFirst [expr $iFL - $O(-near) - [llength $p]] 240 240 241 241 for {set i $iFirst} {$i <= $iFL} {incr i} { 242 - if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i } 242 + set lCand [lrange $col $i [expr $i+$nPm1]] 243 + 244 + set bMatch 1 245 + foreach tok $p term $lCand { 246 + if {[string match $tok $term]==0} { 247 + #puts "$tok $term failed" 248 + set bMatch 0 249 + } 250 + } 251 + if {$bMatch} { 252 + #puts "match at $i" 253 + lappend B($iPhrase) $i 254 + } 255 + 256 + #if {$lCand == $p} { lappend B($iPhrase) $i } 243 257 } 244 258 if {[llength $B($iPhrase)] == 0} break 245 259 } 246 260 247 261 if {$iPhrase==$nPhrase} { 248 262 for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { 249 263 set A($iCol,$iPhrase) [concat $A($iCol,$iPhrase) $B($iPhrase)] ................................................................................ 261 275 foreach a $A($iCol,$iPhrase) { 262 276 lappend res "$counter.$iCol.$a" 263 277 } 264 278 } 265 279 incr counter 266 280 } 267 281 268 - #puts $res 282 + #puts "$aCol -> $res" 269 283 sort_poslist $res 270 284 } 271 285 272 286 #------------------------------------------------------------------------- 273 287 # Usage: 274 288 # 275 289 # sort_poslist LIST
Changes to ext/fts5/test/fts5ac.test.
126 126 } 127 127 128 128 # Argument $expr is an FTS5 match expression designed to be executed against 129 129 # an FTS5 table with the following schema: 130 130 # 131 131 # CREATE VIRTUAL TABLE xy USING fts5(x, y); 132 132 # 133 -# Assuming the table contains the same records as stored int the global 133 +# Assuming the table contains the same records as stored in the global 134 134 # $::data array (see above), this function returns a list containing one 135 135 # element for each match in the dataset. The elements are themselves lists 136 136 # formatted as follows: 137 137 # 138 138 # <rowid> {<phrase 0 matches> <phrase 1 matches>...} 139 139 # 140 140 # where each <phrase X matches> element is a list of phrase matches in the ................................................................................ 171 171 return [concat {*}$res] 172 172 } 173 173 174 174 # 175 175 # End of test code 176 176 #------------------------------------------------------------------------- 177 177 178 -proc fts5_test_poslist {cmd} { 179 - set res [list] 180 - for {set i 0} {$i < [$cmd xInstCount]} {incr i} { 181 - lappend res [string map {{ } .} [$cmd xInst $i]] 182 - } 183 - set res 184 -} 185 - 186 178 187 179 foreach {tn2 sql} { 188 180 1 {} 189 181 2 {BEGIN} 190 182 } { 191 183 reset_db 192 - sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist 184 + fts5_aux_test_functions db 193 185 194 - do_execsql_test 1.0 { 186 + do_execsql_test 1.$tn2.0 { 195 187 CREATE VIRTUAL TABLE xx USING fts5(x,y); 196 188 INSERT INTO xx(xx, rank) VALUES('pgsz', 32); 197 189 } 198 190 199 191 execsql $sql 200 192 201 - do_test $tn2.1.1 { 193 + do_test 1.$tn2.1.1 { 202 194 foreach {id x y} $data { 203 195 execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } 204 196 } 205 197 execsql { INSERT INTO xx(xx) VALUES('integrity-check') } 206 198 } {} 207 199 208 200 ................................................................................ 220 212 8 "c" 221 213 9 "no" 222 214 10 "L O O L V V K" 223 215 } { 224 216 set expr "\"$phrase\"" 225 217 set res [matchdata 1 $expr] 226 218 227 - do_execsql_test $tn2.1.2.$tn.[llength $res] { 219 + do_execsql_test 1.$tn2.1.2.$tn.[llength $res] { 228 220 SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 229 221 } $res 230 222 } 231 223 232 224 #------------------------------------------------------------------------- 233 225 # Test some AND and OR queries. 234 226 # ................................................................................ 242 234 2.2 "a+b OR c" 243 235 2.3 "d+c OR u" 244 236 2.4 "d+c OR u+d" 245 237 246 238 3.1 { a AND b AND c } 247 239 } { 248 240 set res [matchdata 1 $expr] 249 - do_execsql_test $tn2.2.$tn.[llength $res] { 241 + do_execsql_test 1.$tn2.2.$tn.[llength $res] { 250 242 SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 251 243 } $res 252 244 } 253 245 254 246 #------------------------------------------------------------------------- 255 247 # Queries on a specific column. 256 248 # ................................................................................ 271 263 272 264 4.1 {{"x" "y"}:a} 273 265 4.2 {{"y" x}:a} 274 266 4.3 {{x "x"}:b} 275 267 4.4 {{"y" y}:b} 276 268 } { 277 269 set res [matchdata 1 $expr] 278 - do_execsql_test $tn2.3.$tn.[llength $res] { 270 + do_execsql_test 1.$tn2.3.$tn.[llength $res] { 279 271 SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 280 272 } $res 281 273 } 282 274 283 275 #------------------------------------------------------------------------- 284 276 # Some NEAR queries. 285 277 # ................................................................................ 292 284 5 { NEAR(r c, 0) } 293 285 6 { NEAR(a b c) } 294 286 7 { NEAR(a b c, 8) } 295 287 8 { x : NEAR(r c) } 296 288 9 { y : NEAR(r c) } 297 289 } { 298 290 set res [matchdata 1 $expr] 299 - do_execsql_test $tn2.4.1.$tn.[llength $res] { 291 + do_execsql_test 1.$tn2.4.1.$tn.[llength $res] { 300 292 SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr 301 293 } $res 302 294 } 303 295 304 296 do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0} 305 297 do_test $tn2.4.2 { nearset {{a b c}} -- c } {0.0.2} 306 298 307 299 foreach {tn expr tclexpr} { 308 300 1 {a b} {AND [N $x -- {a}] [N $x -- {b}]} 309 301 } { 310 - do_execsql_test $tn2.5.$tn { 302 + do_execsql_test 1.$tn2.5.$tn { 311 303 SELECT fts5_expr_tcl($expr, 'N $x') 312 304 } [list $tclexpr] 313 305 } 314 306 315 307 #------------------------------------------------------------------------- 316 308 # 317 - do_execsql_test $tn2.6.integrity { 309 + do_execsql_test 1.$tn2.6.integrity { 318 310 INSERT INTO xx(xx) VALUES('integrity-check'); 319 311 } 320 312 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} 321 313 foreach {bAsc sql} { 322 314 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} 323 315 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} 324 316 } { ................................................................................ 342 334 15 { a OR b AND c } 343 335 16 { c AND b OR a } 344 336 17 { c AND (b OR a) } 345 337 18 { c NOT (b OR a) } 346 338 19 { c NOT b OR a AND d } 347 339 } { 348 340 set res [matchdata 0 $expr $bAsc] 349 - do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res 341 + do_execsql_test 1.$tn2.6.$bAsc.$tn.[llength $res] $sql $res 350 342 } 351 343 } 352 344 } 353 345 354 -do_execsql_test 3.1 { 346 +do_execsql_test 2.1 { 355 347 SELECT fts5_expr_tcl('a AND b'); 356 348 } {{AND [nearset -- {a}] [nearset -- {b}]}} 349 + 350 +# Some tests for detail=col tables and detail=none. 351 +# 352 +foreach {tn2 sql} { 353 + 1 { 354 + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); 355 + } 356 + 2 { 357 + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=col); 358 + BEGIN; 359 + } 360 + 3 { 361 + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); 362 + BEGIN; 363 + } 364 + 4 { 365 + CREATE VIRTUAL TABLE xx USING fts5(x, y, detail=none); 366 + } 367 +} { 368 + reset_db 369 + fts5_aux_test_functions db 370 + 371 + execsql $sql 372 + 373 + do_execsql_test 3.$tn2.0 { 374 + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); 375 + } 376 + 377 + 378 + do_test 3.$tn2.1.1 { 379 + foreach {id x y} $data { 380 + execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } 381 + } 382 + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } 383 + } {} 384 + 385 + foreach {tn q} { 386 + 1 "o" 2 "b" 3 "e" 4 "m" 5 "l" 6 "a" 7 "b" 8 "c" 9 "no" 10 "L" 387 + 11 "o a" 12 "c AND d" 13 "o OR a" 12 "c OR d" 388 + } { 389 + set res [matchdata 1 $q] 390 + 391 + do_execsql_test 3.$tn2.1.2.$tn.[llength $res] { 392 + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $q 393 + } $res 394 + } 395 + 396 +} 357 397 358 398 finish_test 359 399
Changes to ext/fts5/test/fts5ad.test.
71 71 5 { 72 72 CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); 73 73 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 74 74 BEGIN; 75 75 } 76 76 77 77 6 { 78 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0); 78 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col); 79 79 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 80 80 } 81 81 82 82 7 { 83 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); 83 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); 84 84 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 85 85 } 86 86 87 87 8 { 88 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, offsets=0, prefix="1,2,3,4,5"); 88 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=col, prefix="1,2,3,4,5"); 89 89 INSERT INTO t1(t1, rank) VALUES('pgsz', 32); 90 90 BEGIN; 91 91 } 92 92 93 93 } { 94 94 95 95 do_test $T.1 {
Name change from ext/fts5/test/fts5offsets.test to ext/fts5/test/fts5detail.test.
9 9 # 10 10 #************************************************************************* 11 11 # This file implements regression tests for SQLite library. The 12 12 # focus of this script is testing the FTS5 module. 13 13 # 14 14 15 15 source [file join [file dirname [info script]] fts5_common.tcl] 16 -set testprefix fts5offsets 16 +set testprefix fts5detail 17 17 18 18 # If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 19 ifcapable !fts5 { 20 20 finish_test 21 21 return 22 22 } 23 + 24 +fts5_aux_test_functions db 23 25 24 26 #-------------------------------------------------------------------------- 25 27 # Simple tests. 26 28 # 27 29 do_execsql_test 1.0 { 28 - CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, offsets=0); 30 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col); 29 31 INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1 30 32 INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2 31 33 INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3 32 34 INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4 33 35 INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5 34 36 INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6 35 37 INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7 ................................................................................ 54 56 do_execsql_test 1.2.$tn.2 { 55 57 SELECT rowid FROM t1($match || '*'); 56 58 } $res 57 59 } 58 60 59 61 do_catchsql_test 1.3.1 { 60 62 SELECT rowid FROM t1('h + d'); 61 -} {1 {fts5: phrase queries are not supported (offsets=0)}} 63 +} {1 {fts5: phrase queries are not supported (detail!=full)}} 62 64 63 65 do_catchsql_test 1.3.2 { 64 66 SELECT rowid FROM t1('NEAR(h d)'); 65 -} {1 {fts5: NEAR queries are not supported (offsets=0)}} 67 +} {1 {fts5: NEAR queries are not supported (detail!=full)}} 68 + 66 69 67 70 #------------------------------------------------------------------------- 68 -# integrity-check with both offsets= and prefix= options. 71 +# integrity-check with both detail= and prefix= options. 69 72 # 70 73 do_execsql_test 2.0 { 71 - CREATE VIRTUAL TABLE t2 USING fts5(a, offsets=0, prefix="1"); 74 + CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1"); 72 75 INSERT INTO t2(a) VALUES('aa ab'); 73 76 } 74 77 75 78 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r} 76 79 77 80 do_execsql_test 2.1 { 78 81 INSERT INTO t2(t2) VALUES('integrity-check'); 79 82 } 83 + 84 +do_execsql_test 2.2 { 85 + SELECT fts5_test_poslist(t2) FROM t2('aa'); 86 +} {0.0.0} 87 + 88 +set ::pc 0 89 +#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*] 90 +#exit 80 91 81 92 #------------------------------------------------------------------------- 82 93 # Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs 83 -# work with offsets=0 tables. 94 +# work with detail=col tables. 84 95 # 85 96 set data { 86 97 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} 87 98 2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc} 88 99 3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca} 89 100 4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca} 90 101 5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb} ................................................................................ 120 131 35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb} 121 132 36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa} 122 133 37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb} 123 134 38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac} 124 135 39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba} 125 136 40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca} 126 137 } 138 + 139 +set data { 140 + 1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca} 141 +} 142 + 143 +proc matchdata {expr {bAsc 1}} { 144 + 145 + set tclexpr [db one { 146 + SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z') 147 + }] 148 + set res [list] 149 + 150 + #puts "$expr -> $tclexpr" 151 + foreach {id x y z} $::data { 152 + set cols [list $x $y $z] 153 + set ::pc 0 154 + #set hits [lsort -command instcompare [eval $tclexpr]] 155 + set hits [eval $tclexpr] 156 + if {[llength $hits]>0} { 157 + lappend res [list $id $hits] 158 + } 159 + } 160 + 161 + if {$bAsc} { 162 + set res [lsort -integer -increasing -index 0 $res] 163 + } else { 164 + set res [lsort -integer -decreasing -index 0 $res] 165 + } 166 + 167 + return [concat {*}$res] 168 +} 169 + 127 170 foreach {tn tbl} { 128 - 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, offsets=0) } 171 + 1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) } 129 172 } { 173 +#break 130 174 reset_db 131 175 fts5_aux_test_functions db 132 176 execsql $tbl 133 177 foreach {id x y z} $data { 134 178 execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) } 135 179 } 136 180 foreach {tn2 expr} { 137 181 1 aaa 2 ccc 3 bab 4 aac 138 182 5 aa* 6 cc* 7 ba* 8 aa* 139 183 9 a* 10 b* 11 c* 140 184 } { 141 185 142 - set res [list] 143 - foreach {id x y z} $data { 144 - if {[lsearch [concat $x $y $z] $expr]>=0} { 145 - lappend res $id 146 - set inst [list] 147 - if {[lsearch $x $expr]>=0} { lappend inst 0.0.-1 } 148 - if {[lsearch $y $expr]>=0} { lappend inst 0.1.-1 } 149 - if {[lsearch $z $expr]>=0} { lappend inst 0.2.-1 } 150 - lappend res $inst 151 - } 152 - } 186 + set res [matchdata $expr] 153 187 154 188 do_execsql_test 3.$tn.$tn2.1 { 155 189 SELECT rowid, fts5_test_poslist(t3) FROM t3($expr) 156 190 } $res 157 191 158 192 do_execsql_test 3.$tn.$tn2.2 { 159 193 SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr) 160 194 } $res 161 195 } 196 +} 162 197 198 +#------------------------------------------------------------------------- 199 +# Simple tests for detail=none tables. 200 +# 201 +breakpoint 202 +do_execsql_test 4.0 { 203 + CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none); 204 + INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g'); 205 + INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9'); 163 206 } 164 207 208 +do_catchsql_test 4.1 { 209 + SELECT * FROM t4('a:a') 210 +} {1 {fts5: column queries are not supported (detail=none)}} 211 + 165 212 finish_test 166 213