Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3". |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
aa12f9d9b79c2f523fd6b00e47bcb66d |
| User & Date: | dan 2015-06-09 20:58:39 |
Context
|
2015-06-10
| ||
| 10:45 | Fix the fts5 xRename() method. check-in: 0f7fd513 user: dan tags: fts5 | |
|
2015-06-09
| ||
| 20:58 | Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3". check-in: aa12f9d9 user: dan tags: fts5 | |
|
2015-06-06
| ||
| 19:23 | Fix a comment in fts5.h. check-in: e964b587 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5.c.
1035 1035 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ 1036 1036 int idxNum, /* Strategy index */ 1037 1037 const char *idxStr, /* Unused */ 1038 1038 int nVal, /* Number of elements in apVal */ 1039 1039 sqlite3_value **apVal /* Arguments for the indexing scheme */ 1040 1040 ){ 1041 1041 Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); 1042 + Fts5Config *pConfig = pTab->pConfig; 1042 1043 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 1043 1044 int rc = SQLITE_OK; /* Error code */ 1044 1045 int iVal = 0; /* Counter for apVal[] */ 1045 1046 int bDesc; /* True if ORDER BY [rank|rowid] DESC */ 1046 1047 int bOrderByRank; /* True if ORDER BY rank */ 1047 1048 sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ 1048 1049 sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ 1049 1050 sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ 1050 1051 sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ 1051 1052 sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ 1052 - char **pzErrmsg = pTab->pConfig->pzErrmsg; 1053 + char **pzErrmsg = pConfig->pzErrmsg; 1053 1054 1054 1055 assert( pCsr->pStmt==0 ); 1055 1056 assert( pCsr->pExpr==0 ); 1056 1057 assert( pCsr->csrflags==0 ); 1057 1058 assert( pCsr->pRank==0 ); 1058 1059 assert( pCsr->zRank==0 ); 1059 1060 assert( pCsr->zRankArgs==0 ); 1060 1061 1061 1062 assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); 1062 - pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; 1063 + pConfig->pzErrmsg = &pTab->base.zErrMsg; 1063 1064 1064 1065 /* Decode the arguments passed through to this function. 1065 1066 ** 1066 1067 ** Note: The following set of if(...) statements must be in the same 1067 1068 ** order as the corresponding entries in the struct at the top of 1068 1069 ** fts5BestIndexMethod(). */ 1069 1070 if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; ................................................................................ 1103 1104 assert( pCsr->iFirstRowid==SMALLEST_INT64 ); 1104 1105 pCsr->ePlan = FTS5_PLAN_SOURCE; 1105 1106 pCsr->pExpr = pTab->pSortCsr->pExpr; 1106 1107 rc = fts5CursorFirst(pTab, pCsr, bDesc); 1107 1108 }else if( pMatch ){ 1108 1109 const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 1109 1110 1110 - rc = fts5CursorParseRank(pTab->pConfig, pCsr, pRank); 1111 + rc = fts5CursorParseRank(pConfig, pCsr, pRank); 1111 1112 if( rc==SQLITE_OK ){ 1112 1113 if( zExpr[0]=='*' ){ 1113 1114 /* The user has issued a query of the form "MATCH '*...'". This 1114 1115 ** indicates that the MATCH expression is not a full text query, 1115 1116 ** but a request for an internal parameter. */ 1116 1117 rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); 1117 1118 }else{ 1118 1119 char **pzErr = &pTab->base.zErrMsg; 1119 - rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr); 1120 + rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr); 1120 1121 if( rc==SQLITE_OK ){ 1121 1122 if( bOrderByRank ){ 1122 1123 pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; 1123 1124 rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); 1124 1125 }else{ 1125 1126 pCsr->ePlan = FTS5_PLAN_MATCH; 1126 1127 rc = fts5CursorFirst(pTab, pCsr, bDesc); 1127 1128 } 1128 1129 } 1129 1130 } 1130 1131 } 1132 + }else if( pConfig->zContent==0 ){ 1133 + *pConfig->pzErrmsg = sqlite3_mprintf( 1134 + "%s: table does not support scanning", pConfig->zName 1135 + ); 1136 + rc = SQLITE_ERROR; 1131 1137 }else{ 1132 1138 /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup 1133 1139 ** by rowid (ePlan==FTS5_PLAN_ROWID). */ 1134 1140 pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); 1135 1141 rc = sqlite3Fts5StorageStmt( 1136 1142 pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg 1137 1143 ); ................................................................................ 1142 1148 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); 1143 1149 sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); 1144 1150 } 1145 1151 rc = fts5NextMethod(pCursor); 1146 1152 } 1147 1153 } 1148 1154 1149 - pTab->pConfig->pzErrmsg = pzErrmsg; 1155 + pConfig->pzErrmsg = pzErrmsg; 1150 1156 return rc; 1151 1157 } 1152 1158 1153 1159 /* 1154 1160 ** This is the xEof method of the virtual table. SQLite calls this 1155 1161 ** routine to find out if it has reached the end of a result set. 1156 1162 */ ................................................................................ 1616 1622 if( rc==SQLITE_OK ){ 1617 1623 *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); 1618 1624 *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); 1619 1625 } 1620 1626 } 1621 1627 return rc; 1622 1628 } 1629 + 1630 +static int fts5ColumnSizeCb( 1631 + void *pContext, /* Pointer to int */ 1632 + const char *pToken, /* Buffer containing token */ 1633 + int nToken, /* Size of token in bytes */ 1634 + int iStart, /* Start offset of token */ 1635 + int iEnd /* End offset of token */ 1636 +){ 1637 + int *pCnt = (int*)pContext; 1638 + *pCnt = *pCnt + 1; 1639 + return SQLITE_OK; 1640 +} 1623 1641 1624 1642 static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ 1625 1643 Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; 1626 1644 Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); 1645 + Fts5Config *pConfig = pTab->pConfig; 1627 1646 int rc = SQLITE_OK; 1628 1647 1629 1648 if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ 1630 - i64 iRowid = fts5CursorRowid(pCsr); 1631 - rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); 1649 + if( pConfig->bColumnsize ){ 1650 + i64 iRowid = fts5CursorRowid(pCsr); 1651 + rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); 1652 + }else if( pConfig->zContent==0 ){ 1653 + int i; 1654 + for(i=0; i<pConfig->nCol; i++){ 1655 + if( pConfig->abUnindexed[i]==0 ){ 1656 + pCsr->aColumnSize[i] = -1; 1657 + } 1658 + } 1659 + }else{ 1660 + int i; 1661 + for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){ 1662 + if( pConfig->abUnindexed[i]==0 ){ 1663 + const char *z; int n; 1664 + void *p = (void*)(&pCsr->aColumnSize[i]); 1665 + pCsr->aColumnSize[i] = 0; 1666 + rc = fts5ApiColumnText(pCtx, i, &z, &n); 1667 + if( rc==SQLITE_OK ){ 1668 + rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb); 1669 + } 1670 + } 1671 + } 1672 + } 1632 1673 CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE); 1633 1674 } 1634 1675 if( iCol<0 ){ 1635 1676 int i; 1636 1677 *pnToken = 0; 1637 - for(i=0; i<pTab->pConfig->nCol; i++){ 1678 + for(i=0; i<pConfig->nCol; i++){ 1638 1679 *pnToken += pCsr->aColumnSize[i]; 1639 1680 } 1640 - }else if( iCol<pTab->pConfig->nCol ){ 1681 + }else if( iCol<pConfig->nCol ){ 1641 1682 *pnToken = pCsr->aColumnSize[iCol]; 1642 1683 }else{ 1643 1684 *pnToken = 0; 1644 1685 rc = SQLITE_RANGE; 1645 1686 } 1646 1687 return rc; 1647 1688 } ................................................................................ 1952 1993 } 1953 1994 1954 1995 /* No function of the specified name was found. Return 0. */ 1955 1996 return 0; 1956 1997 } 1957 1998 1958 1999 /* 1959 -** Implementation of FTS3 xRename method. Rename an fts5 table. 2000 +** Implementation of FTS5 xRename method. Rename an fts5 table. 1960 2001 */ 1961 2002 static int fts5RenameMethod( 1962 2003 sqlite3_vtab *pVtab, /* Virtual table handle */ 1963 2004 const char *zName /* New name of table */ 1964 2005 ){ 1965 2006 int rc = SQLITE_OK; 1966 2007 return rc;
Changes to ext/fts5/fts5Int.h.
101 101 ** zContentExprlist: 102 102 ** 103 103 ** pzErrmsg: 104 104 ** This exists in order to allow the fts5_index.c module to return a 105 105 ** decent error message if it encounters a file-format version it does 106 106 ** not understand. 107 107 ** 108 +** bColumnsize: 109 +** True if the %_docsize table is created. 110 +** 108 111 */ 109 112 struct Fts5Config { 110 113 sqlite3 *db; /* Database handle */ 111 114 char *zDb; /* Database holding FTS index (e.g. "main") */ 112 115 char *zName; /* Name of FTS index */ 113 116 int nCol; /* Number of columns */ 114 117 char **azCol; /* Column names */ 115 118 u8 *abUnindexed; /* True for unindexed columns */ 116 119 int nPrefix; /* Number of prefix indexes */ 117 120 int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ 118 121 int eContent; /* An FTS5_CONTENT value */ 119 122 char *zContent; /* content table */ 120 123 char *zContentRowid; /* "content_rowid=" option value */ 124 + int bColumnsize; /* "columnsize=" option value (dflt==1) */ 121 125 char *zContentExprlist; 122 126 Fts5Tokenizer *pTok; 123 127 fts5_tokenizer *pTokApi; 124 128 125 129 /* Values loaded from the %_config table */ 126 130 int iCookie; /* Incremented when %_config is modified */ 127 131 int pgsz; /* Approximate page size used in %_data */ ................................................................................ 191 195 void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*); 192 196 void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*); 193 197 void sqlite3Fts5BufferFree(Fts5Buffer*); 194 198 void sqlite3Fts5BufferZero(Fts5Buffer*); 195 199 void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*); 196 200 void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); 197 201 void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int); 202 + 203 +char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...); 198 204 199 205 #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) 200 206 #define fts5BufferGrow(a,b,c) sqlite3Fts5BufferGrow(a,b,c) 201 207 #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) 202 208 #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) 203 209 #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) 204 210 #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d)
Changes to ext/fts5/fts5_buffer.c.
120 120 *pRc = SQLITE_NOMEM; 121 121 }else{ 122 122 sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp); 123 123 sqlite3_free(zTmp); 124 124 } 125 125 } 126 126 } 127 + 128 +char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...){ 129 + char *zRet = 0; 130 + if( *pRc==SQLITE_OK ){ 131 + va_list ap; 132 + va_start(ap, zFmt); 133 + zRet = sqlite3_vmprintf(zFmt, ap); 134 + va_end(ap); 135 + if( zRet==0 ){ 136 + *pRc = SQLITE_NOMEM; 137 + } 138 + } 139 + return zRet; 140 +} 141 + 127 142 128 143 /* 129 144 ** Free any buffer allocated by pBuf. Zero the structure before returning. 130 145 */ 131 146 void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ 132 147 sqlite3_free(pBuf->p); 133 148 memset(pBuf, 0, sizeof(Fts5Buffer));
Changes to ext/fts5/fts5_config.c.
192 192 quote = z[0]; 193 193 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ 194 194 fts5Dequote(z); 195 195 } 196 196 } 197 197 198 198 /* 199 -** Parse the "special" CREATE VIRTUAL TABLE directive and update 199 +** Parse a "special" CREATE VIRTUAL TABLE directive and update 200 200 ** configuration object pConfig as appropriate. 201 201 ** 202 202 ** If successful, object pConfig is updated and SQLITE_OK returned. If 203 203 ** an error occurs, an SQLite error code is returned and an error message 204 204 ** may be left in *pzErr. It is the responsibility of the caller to 205 205 ** eventually free any such error message using sqlite3_free(). 206 206 */ ................................................................................ 207 207 static int fts5ConfigParseSpecial( 208 208 Fts5Global *pGlobal, 209 209 Fts5Config *pConfig, /* Configuration object to update */ 210 210 const char *zCmd, /* Special command to parse */ 211 211 const char *zArg, /* Argument to parse */ 212 212 char **pzErr /* OUT: Error message */ 213 213 ){ 214 + int rc = SQLITE_OK; 214 215 int nCmd = strlen(zCmd); 215 216 if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ 216 217 const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; 217 - int rc = SQLITE_OK; 218 218 const char *p; 219 219 if( pConfig->aPrefix ){ 220 220 *pzErr = sqlite3_mprintf("multiple prefix=... directives"); 221 221 rc = SQLITE_ERROR; 222 222 }else{ 223 223 pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte); 224 224 } ................................................................................ 244 244 pConfig->aPrefix[pConfig->nPrefix] = nPre; 245 245 pConfig->nPrefix++; 246 246 } 247 247 return rc; 248 248 } 249 249 250 250 if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ 251 - int rc = SQLITE_OK; 252 251 const char *p = (const char*)zArg; 253 252 int nArg = strlen(zArg) + 1; 254 253 char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); 255 254 char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); 256 255 char *pSpace = pDel; 257 256 258 257 if( azArg && pSpace ){ ................................................................................ 289 288 290 289 sqlite3_free(azArg); 291 290 sqlite3_free(pDel); 292 291 return rc; 293 292 } 294 293 295 294 if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){ 296 - int rc = SQLITE_OK; 297 295 if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ 298 296 *pzErr = sqlite3_mprintf("multiple content=... directives"); 299 297 rc = SQLITE_ERROR; 300 298 }else{ 301 299 if( zArg[0] ){ 302 300 pConfig->eContent = FTS5_CONTENT_EXTERNAL; 303 301 pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg); 302 + if( pConfig->zContent==0 ) rc = SQLITE_NOMEM; 304 303 }else{ 305 304 pConfig->eContent = FTS5_CONTENT_NONE; 306 - pConfig->zContent = sqlite3_mprintf( 307 - "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName 308 - ); 309 305 } 310 - if( pConfig->zContent==0 ) rc = SQLITE_NOMEM; 311 306 } 312 307 return rc; 313 308 } 314 309 315 310 if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ 316 - int rc = SQLITE_OK; 317 311 if( pConfig->zContentRowid ){ 318 312 *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); 319 313 rc = SQLITE_ERROR; 320 314 }else{ 321 315 pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1); 322 316 } 323 317 return rc; 324 318 } 319 + 320 + if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){ 321 + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ 322 + *pzErr = sqlite3_mprintf("malformed columnsize=... directive"); 323 + rc = SQLITE_ERROR; 324 + }else{ 325 + pConfig->bColumnsize = (zArg[0]=='1'); 326 + } 327 + return rc; 328 + } 325 329 326 330 *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); 327 331 return SQLITE_ERROR; 328 332 } 329 333 330 334 /* 331 335 ** Allocate an instance of the default tokenizer ("simple") at ................................................................................ 473 477 pRet->iCookie = -1; 474 478 475 479 nByte = nArg * (sizeof(char*) + sizeof(u8)); 476 480 pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); 477 481 pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; 478 482 pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); 479 483 pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); 484 + pRet->bColumnsize = 1; 480 485 if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){ 481 486 *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName); 482 487 rc = SQLITE_ERROR; 483 488 } 484 489 485 490 for(i=3; rc==SQLITE_OK && i<nArg; i++){ 486 491 const char *zOrig = azArg[i]; ................................................................................ 526 531 ** already been allocated. Otherwise, allocate an instance of the default 527 532 ** tokenizer (unicode61) now. */ 528 533 if( rc==SQLITE_OK && pRet->pTok==0 ){ 529 534 rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); 530 535 } 531 536 532 537 /* If no zContent option was specified, fill in the default values. */ 533 - if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){ 534 - pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName); 535 - if( pRet->zContent==0 ){ 536 - rc = SQLITE_NOMEM; 537 - }else{ 538 - sqlite3_free(pRet->zContentRowid); 539 - pRet->zContentRowid = 0; 538 + if( rc==SQLITE_OK && pRet->zContent==0 ){ 539 + const char *zTail = 0; 540 + assert( pRet->eContent==FTS5_CONTENT_NORMAL 541 + || pRet->eContent==FTS5_CONTENT_NONE 542 + ); 543 + if( pRet->eContent==FTS5_CONTENT_NORMAL ){ 544 + zTail = "content"; 545 + }else if( pRet->bColumnsize ){ 546 + zTail = "docsize"; 547 + } 548 + 549 + if( zTail ){ 550 + pRet->zContent = sqlite3Fts5Mprintf( 551 + &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail 552 + ); 540 553 } 541 554 } 555 + 542 556 if( rc==SQLITE_OK && pRet->zContentRowid==0 ){ 543 557 pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1); 544 558 } 545 559 546 560 /* Formulate the zContentExprlist text */ 547 561 if( rc==SQLITE_OK ){ 548 562 rc = fts5ConfigMakeExprlist(pRet);
Changes to ext/fts5/fts5_storage.c.
35 35 #endif 36 36 #if FTS5_STMT_LOOKUP!=2 37 37 # error "FTS5_STMT_LOOKUP mismatch" 38 38 #endif 39 39 40 40 #define FTS5_STMT_INSERT_CONTENT 3 41 41 #define FTS5_STMT_REPLACE_CONTENT 4 42 - 43 42 #define FTS5_STMT_DELETE_CONTENT 5 44 43 #define FTS5_STMT_REPLACE_DOCSIZE 6 45 44 #define FTS5_STMT_DELETE_DOCSIZE 7 46 - 47 45 #define FTS5_STMT_LOOKUP_DOCSIZE 8 48 - 49 46 #define FTS5_STMT_REPLACE_CONFIG 9 50 - 51 47 #define FTS5_STMT_SCAN 10 52 48 53 49 /* 54 50 ** Prepare the two insert statements - Fts5Storage.pInsertContent and 55 51 ** Fts5Storage.pInsertDocsize - if they have not already been prepared. 56 52 ** Return SQLITE_OK if successful, or an SQLite error code if an error 57 53 ** occurs. ................................................................................ 59 55 static int fts5StorageGetStmt( 60 56 Fts5Storage *p, /* Storage handle */ 61 57 int eStmt, /* FTS5_STMT_XXX constant */ 62 58 sqlite3_stmt **ppStmt, /* OUT: Prepared statement handle */ 63 59 char **pzErrMsg /* OUT: Error message (if any) */ 64 60 ){ 65 61 int rc = SQLITE_OK; 62 + 63 + /* If there is no %_docsize table, there should be no requests for 64 + ** statements to operate on it. */ 65 + assert( p->pConfig->bColumnsize || ( 66 + eStmt!=FTS5_STMT_REPLACE_DOCSIZE 67 + && eStmt!=FTS5_STMT_DELETE_DOCSIZE 68 + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE 69 + )); 66 70 67 71 assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) ); 68 72 if( p->aStmt[eStmt]==0 ){ 69 73 const char *azStmt[] = { 70 74 "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", 71 75 "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", 72 76 "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ ................................................................................ 171 175 /* 172 176 ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error 173 177 ** code otherwise. 174 178 */ 175 179 int sqlite3Fts5DropAll(Fts5Config *pConfig){ 176 180 int rc = fts5ExecPrintf(pConfig->db, 0, 177 181 "DROP TABLE IF EXISTS %Q.'%q_data';" 178 - "DROP TABLE IF EXISTS %Q.'%q_docsize';" 179 182 "DROP TABLE IF EXISTS %Q.'%q_config';", 180 - pConfig->zDb, pConfig->zName, 181 183 pConfig->zDb, pConfig->zName, 182 184 pConfig->zDb, pConfig->zName 183 185 ); 186 + if( rc==SQLITE_OK && pConfig->bColumnsize ){ 187 + rc = fts5ExecPrintf(pConfig->db, 0, 188 + "DROP TABLE IF EXISTS %Q.'%q_docsize';", 189 + pConfig->zDb, pConfig->zName 190 + ); 191 + } 184 192 if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ 185 193 rc = fts5ExecPrintf(pConfig->db, 0, 186 194 "DROP TABLE IF EXISTS %Q.'%q_content';", 187 195 pConfig->zDb, pConfig->zName 188 196 ); 189 197 } 190 198 return rc; ................................................................................ 262 270 iOff += strlen(&zDefn[iOff]); 263 271 } 264 272 rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); 265 273 } 266 274 sqlite3_free(zDefn); 267 275 } 268 276 269 - if( rc==SQLITE_OK ){ 277 + if( rc==SQLITE_OK && pConfig->bColumnsize ){ 270 278 rc = sqlite3Fts5CreateTable( 271 279 pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr 272 280 ); 273 281 } 274 282 if( rc==SQLITE_OK ){ 275 283 rc = sqlite3Fts5CreateTable( 276 284 pConfig, "config", "k PRIMARY KEY, v", 1, pzErr ................................................................................ 370 378 } 371 379 372 380 373 381 /* 374 382 ** Insert a record into the %_docsize table. Specifically, do: 375 383 ** 376 384 ** INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf); 385 +** 386 +** If there is no %_docsize table (as happens if the columnsize=0 option 387 +** is specified when the FTS5 table is created), this function is a no-op. 377 388 */ 378 389 static int fts5StorageInsertDocsize( 379 390 Fts5Storage *p, /* Storage module to write to */ 380 391 i64 iRowid, /* id value */ 381 392 Fts5Buffer *pBuf /* sz value */ 382 393 ){ 383 - sqlite3_stmt *pReplace = 0; 384 - int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 385 - if( rc==SQLITE_OK ){ 386 - sqlite3_bind_int64(pReplace, 1, iRowid); 387 - sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); 388 - sqlite3_step(pReplace); 389 - rc = sqlite3_reset(pReplace); 394 + int rc = SQLITE_OK; 395 + if( p->pConfig->bColumnsize ){ 396 + sqlite3_stmt *pReplace = 0; 397 + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 398 + if( rc==SQLITE_OK ){ 399 + sqlite3_bind_int64(pReplace, 1, iRowid); 400 + sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); 401 + sqlite3_step(pReplace); 402 + rc = sqlite3_reset(pReplace); 403 + } 390 404 } 391 405 return rc; 392 406 } 393 407 394 408 /* 395 409 ** Load the contents of the "averages" record from disk into the 396 410 ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if ................................................................................ 451 465 return rc; 452 466 } 453 467 454 468 /* 455 469 ** Remove a row from the FTS table. 456 470 */ 457 471 int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){ 472 + Fts5Config *pConfig = p->pConfig; 458 473 int rc; 459 474 sqlite3_stmt *pDel; 460 475 461 476 rc = fts5StorageLoadTotals(p, 1); 462 477 463 478 /* Delete the index records */ 464 479 if( rc==SQLITE_OK ){ 465 480 rc = fts5StorageDeleteFromIndex(p, iDel); 466 481 } 467 482 468 483 /* Delete the %_docsize record */ 469 - if( rc==SQLITE_OK ){ 484 + if( rc==SQLITE_OK && pConfig->bColumnsize ){ 470 485 rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); 471 486 } 472 487 if( rc==SQLITE_OK ){ 473 488 sqlite3_bind_int64(pDel, 1, iDel); 474 489 sqlite3_step(pDel); 475 490 rc = sqlite3_reset(pDel); 476 491 } ................................................................................ 524 539 ); 525 540 p->aTotalSize[iCol] -= (i64)ctx.szCol; 526 541 } 527 542 p->nTotalRow--; 528 543 } 529 544 530 545 /* Delete the %_docsize record */ 531 - if( rc==SQLITE_OK ){ 532 - rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); 533 - } 534 - if( rc==SQLITE_OK ){ 535 - sqlite3_bind_int64(pDel, 1, iDel); 536 - sqlite3_step(pDel); 537 - rc = sqlite3_reset(pDel); 546 + if( pConfig->bColumnsize ){ 547 + if( rc==SQLITE_OK ){ 548 + rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0); 549 + } 550 + if( rc==SQLITE_OK ){ 551 + sqlite3_bind_int64(pDel, 1, iDel); 552 + sqlite3_step(pDel); 553 + rc = sqlite3_reset(pDel); 554 + } 538 555 } 539 556 540 557 /* Write the averages record */ 541 558 if( rc==SQLITE_OK ){ 542 559 rc = fts5StorageSaveTotals(p); 543 560 } 544 561 ................................................................................ 550 567 */ 551 568 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ 552 569 Fts5Config *pConfig = p->pConfig; 553 570 int rc; 554 571 555 572 /* Delete the contents of the %_data and %_docsize tables. */ 556 573 rc = fts5ExecPrintf(pConfig->db, 0, 557 - "DELETE FROM %Q.'%q_data';" 558 - "DELETE FROM %Q.'%q_docsize';", 559 - pConfig->zDb, pConfig->zName, 574 + "DELETE FROM %Q.'%q_data';", 560 575 pConfig->zDb, pConfig->zName 561 576 ); 577 + if( rc==SQLITE_OK && pConfig->bColumnsize ){ 578 + rc = fts5ExecPrintf(pConfig->db, 0, 579 + "DELETE FROM %Q.'%q_docsize';", 580 + pConfig->zDb, pConfig->zName 581 + ); 582 + } 562 583 563 584 /* Reinitialize the %_data table. This call creates the initial structure 564 585 ** and averages records. */ 565 586 if( rc==SQLITE_OK ){ 566 587 rc = sqlite3Fts5IndexReinit(p->pIndex); 567 588 } 568 589 if( rc==SQLITE_OK ){ ................................................................................ 631 652 } 632 653 633 654 /* 634 655 ** Allocate a new rowid. This is used for "external content" tables when 635 656 ** a NULL value is inserted into the rowid column. The new rowid is allocated 636 657 ** by inserting a dummy row into the %_docsize table. The dummy will be 637 658 ** overwritten later. 659 +** 660 +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In 661 +** this case the user is required to provide a rowid explicitly. 638 662 */ 639 663 static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ 640 - sqlite3_stmt *pReplace = 0; 641 - int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 642 - if( rc==SQLITE_OK ){ 643 - sqlite3_bind_null(pReplace, 1); 644 - sqlite3_bind_null(pReplace, 2); 645 - sqlite3_step(pReplace); 646 - rc = sqlite3_reset(pReplace); 647 - } 648 - if( rc==SQLITE_OK ){ 649 - *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); 664 + int rc = SQLITE_MISMATCH; 665 + if( p->pConfig->bColumnsize ){ 666 + sqlite3_stmt *pReplace = 0; 667 + rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); 668 + if( rc==SQLITE_OK ){ 669 + sqlite3_bind_null(pReplace, 1); 670 + sqlite3_bind_null(pReplace, 2); 671 + sqlite3_step(pReplace); 672 + rc = sqlite3_reset(pReplace); 673 + } 674 + if( rc==SQLITE_OK ){ 675 + *piRowid = sqlite3_last_insert_rowid(p->pConfig->db); 676 + } 650 677 } 651 678 return rc; 652 679 } 653 680 654 681 /* 655 682 ** Insert a new row into the FTS table. 656 683 */ ................................................................................ 954 981 } 955 982 } 956 983 rc = sqlite3_reset(pLookup); 957 984 if( bCorrupt && rc==SQLITE_OK ){ 958 985 rc = FTS5_CORRUPT; 959 986 } 960 987 } 988 + 961 989 return rc; 962 990 } 963 991 964 992 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ 965 993 int rc = fts5StorageLoadTotals(p, 0); 966 994 if( rc==SQLITE_OK ){ 967 995 *pnToken = 0;
Changes to ext/fts5/test/fts5ae.test.
144 144 ORDER BY rowid DESC; 145 145 } { 146 146 3 {1 0} 147 147 2 {0 1} 148 148 1 {4 6} 149 149 } 150 150 151 -do_execsql_test 5.2 { 151 +do_execsql_test 5.3 { 152 152 SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a' 153 153 ORDER BY rowid DESC; 154 154 } { 155 155 3 {a {}} 156 156 2 {{} a} 157 157 1 {{a b c d} {e f g h i j}} 158 158 } 159 159 160 -do_execsql_test 5.3 { 160 +do_execsql_test 5.4 { 161 161 SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a' 162 162 ORDER BY rowid DESC; 163 163 } { 164 164 3 {5 7} 165 165 2 {5 7} 166 166 1 {5 7} 167 167 } 168 168 169 -do_execsql_test 5.4 { 169 +do_execsql_test 5.5 { 170 170 INSERT INTO t5 VALUES('x y z', 'v w x y z'); 171 171 SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a' 172 172 ORDER BY rowid DESC; 173 173 } { 174 174 3 {8 12} 175 175 2 {8 12} 176 176 1 {8 12}
Added ext/fts5/test/fts5columnsize.test.
1 +# 2015 Jun 10 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# Tests focusing on fts5 tables with the columnsize=0 option. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5columnsize 17 + 18 +#------------------------------------------------------------------------- 19 +# Check that the option can be parsed and that the %_docsize table is 20 +# only created if it is set to true. 21 +# 22 +foreach {tn outcome stmt} { 23 + 1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) } 24 + 2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) } 25 + 3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') } 26 + 4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') } 27 + 5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') } 28 + 6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) } 29 + 7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) } 30 + 8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) } 31 +} { 32 + execsql { 33 + DROP TABLE IF EXISTS t1; 34 + } 35 + if {$outcome==2} { 36 + do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}} 37 + } else { 38 + do_execsql_test 1.$tn.2 $stmt 39 + do_execsql_test 1.$tn.3 { 40 + SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize' 41 + } $outcome 42 + } 43 +} 44 + 45 +#------------------------------------------------------------------------- 46 +# Run tests on a table with no %_content or %_docsize backing store. 47 +# 48 +do_execsql_test 2.0 { 49 + CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content=''); 50 +} 51 +do_catchsql_test 2.1 { 52 + INSERT INTO t2 VALUES('a b c d e f'); 53 +} {1 {datatype mismatch}} 54 +do_execsql_test 2.2 { 55 + INSERT INTO t2(rowid, x) VALUES(1, 'c d e f'); 56 + INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h'); 57 + INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h'); 58 +} {} 59 +do_execsql_test 2.3 { 60 + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; 61 + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; 62 + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 63 +} {3 :: 1 2 3 :: 2 3} 64 +do_execsql_test 2.4 { 65 + INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h'); 66 + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; 67 + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; 68 + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 69 +} {3 :: 1 3 :: 3} 70 +do_execsql_test 2.5 { 71 + INSERT INTO t2(t2) VALUES('delete-all'); 72 + SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::'; 73 + SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::'; 74 + SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 75 +} {:: ::} 76 +do_execsql_test 2.6 { 77 + INSERT INTO t2(rowid, x) VALUES(1, 'o t t f'); 78 + INSERT INTO t2(rowid, x) VALUES(2, 'f s s e'); 79 + INSERT INTO t2(rowid, x) VALUES(3, 'n t e t'); 80 +} 81 + 82 +do_catchsql_test 2.7.1 { 83 + SELECT rowid FROM t2 84 +} {1 {t2: table does not support scanning}} 85 +do_catchsql_test 2.7.2 { 86 + SELECT rowid FROM t2 WHERE rowid=2 87 +} {1 {t2: table does not support scanning}} 88 +do_catchsql_test 2.7.3 { 89 + SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3 90 +} {1 {t2: table does not support scanning}} 91 + 92 +do_execsql_test 2.X { 93 + DROP TABLE t2 94 +} 95 + 96 +#------------------------------------------------------------------------- 97 +# Test the xColumnSize() API 98 +# 99 +fts5_aux_test_functions db 100 + 101 +do_execsql_test 3.0 { 102 + CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0); 103 + INSERT INTO t3 VALUES('a a', 'b b b', 'c'); 104 + INSERT INTO t3 VALUES('x a x', 'b b b y', ''); 105 +} 106 +do_execsql_test 3.1 { 107 + SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a' 108 +} { 109 + 1 {2 0 1} 2 {3 0 0} 110 +} 111 + 112 +finish_test