/ Check-in [aa12f9d9]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3".
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: aa12f9d9b79c2f523fd6b00e47bcb66dba09ce0c
User & Date: dan 2015-06-09 20:58:39
Context
2015-06-10
10:45
Fix the fts5 xRename() method. check-in: 0f7fd513 user: dan tags: fts5
2015-06-09
20:58
Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3". check-in: aa12f9d9 user: dan tags: fts5
2015-06-06
19:23
Fix a comment in fts5.h. check-in: e964b587 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

  1035   1035     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  1036   1036     int idxNum,                     /* Strategy index */
  1037   1037     const char *idxStr,             /* Unused */
  1038   1038     int nVal,                       /* Number of elements in apVal */
  1039   1039     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  1040   1040   ){
  1041   1041     Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
         1042  +  Fts5Config *pConfig = pTab->pConfig;
  1042   1043     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  1043   1044     int rc = SQLITE_OK;             /* Error code */
  1044   1045     int iVal = 0;                   /* Counter for apVal[] */
  1045   1046     int bDesc;                      /* True if ORDER BY [rank|rowid] DESC */
  1046   1047     int bOrderByRank;               /* True if ORDER BY rank */
  1047   1048     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1048   1049     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1049   1050     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1050   1051     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1051   1052     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  1052         -  char **pzErrmsg = pTab->pConfig->pzErrmsg;
         1053  +  char **pzErrmsg = pConfig->pzErrmsg;
  1053   1054   
  1054   1055     assert( pCsr->pStmt==0 );
  1055   1056     assert( pCsr->pExpr==0 );
  1056   1057     assert( pCsr->csrflags==0 );
  1057   1058     assert( pCsr->pRank==0 );
  1058   1059     assert( pCsr->zRank==0 );
  1059   1060     assert( pCsr->zRankArgs==0 );
  1060   1061   
  1061   1062     assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg );
  1062         -  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
         1063  +  pConfig->pzErrmsg = &pTab->base.zErrMsg;
  1063   1064   
  1064   1065     /* Decode the arguments passed through to this function.
  1065   1066     **
  1066   1067     ** Note: The following set of if(...) statements must be in the same
  1067   1068     ** order as the corresponding entries in the struct at the top of
  1068   1069     ** fts5BestIndexMethod().  */
  1069   1070     if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
................................................................................
  1103   1104       assert( pCsr->iFirstRowid==SMALLEST_INT64 );
  1104   1105       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1105   1106       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1106   1107       rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1107   1108     }else if( pMatch ){
  1108   1109       const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1109   1110   
  1110         -    rc = fts5CursorParseRank(pTab->pConfig, pCsr, pRank);
         1111  +    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1111   1112       if( rc==SQLITE_OK ){
  1112   1113         if( zExpr[0]=='*' ){
  1113   1114           /* The user has issued a query of the form "MATCH '*...'". This
  1114   1115           ** indicates that the MATCH expression is not a full text query,
  1115   1116           ** but a request for an internal parameter.  */
  1116   1117           rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
  1117   1118         }else{
  1118   1119           char **pzErr = &pTab->base.zErrMsg;
  1119         -        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
         1120  +        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
  1120   1121           if( rc==SQLITE_OK ){
  1121   1122             if( bOrderByRank ){
  1122   1123               pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
  1123   1124               rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
  1124   1125             }else{
  1125   1126               pCsr->ePlan = FTS5_PLAN_MATCH;
  1126   1127               rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1127   1128             }
  1128   1129           }
  1129   1130         }
  1130   1131       }
         1132  +  }else if( pConfig->zContent==0 ){
         1133  +    *pConfig->pzErrmsg = sqlite3_mprintf(
         1134  +        "%s: table does not support scanning", pConfig->zName
         1135  +    );
         1136  +    rc = SQLITE_ERROR;
  1131   1137     }else{
  1132   1138       /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
  1133   1139       ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
  1134   1140       pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN);
  1135   1141       rc = sqlite3Fts5StorageStmt(
  1136   1142           pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg
  1137   1143       );
................................................................................
  1142   1148           sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
  1143   1149           sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
  1144   1150         }
  1145   1151         rc = fts5NextMethod(pCursor);
  1146   1152       }
  1147   1153     }
  1148   1154   
  1149         -  pTab->pConfig->pzErrmsg = pzErrmsg;
         1155  +  pConfig->pzErrmsg = pzErrmsg;
  1150   1156     return rc;
  1151   1157   }
  1152   1158   
  1153   1159   /* 
  1154   1160   ** This is the xEof method of the virtual table. SQLite calls this 
  1155   1161   ** routine to find out if it has reached the end of a result set.
  1156   1162   */
................................................................................
  1616   1622       if( rc==SQLITE_OK ){
  1617   1623         *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
  1618   1624         *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
  1619   1625       }
  1620   1626     }
  1621   1627     return rc;
  1622   1628   }
         1629  +
         1630  +static int fts5ColumnSizeCb(
         1631  +  void *pContext,                 /* Pointer to int */
         1632  +  const char *pToken,             /* Buffer containing token */
         1633  +  int nToken,                     /* Size of token in bytes */
         1634  +  int iStart,                     /* Start offset of token */
         1635  +  int iEnd                        /* End offset of token */
         1636  +){
         1637  +  int *pCnt = (int*)pContext;
         1638  +  *pCnt = *pCnt + 1;
         1639  +  return SQLITE_OK;
         1640  +}
  1623   1641   
  1624   1642   static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
  1625   1643     Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  1626   1644     Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
         1645  +  Fts5Config *pConfig = pTab->pConfig;
  1627   1646     int rc = SQLITE_OK;
  1628   1647   
  1629   1648     if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
  1630         -    i64 iRowid = fts5CursorRowid(pCsr);
  1631         -    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
         1649  +    if( pConfig->bColumnsize ){
         1650  +      i64 iRowid = fts5CursorRowid(pCsr);
         1651  +      rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
         1652  +    }else if( pConfig->zContent==0 ){
         1653  +      int i;
         1654  +      for(i=0; i<pConfig->nCol; i++){
         1655  +        if( pConfig->abUnindexed[i]==0 ){
         1656  +          pCsr->aColumnSize[i] = -1;
         1657  +        }
         1658  +      }
         1659  +    }else{
         1660  +      int i;
         1661  +      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
         1662  +        if( pConfig->abUnindexed[i]==0 ){
         1663  +          const char *z; int n;
         1664  +          void *p = (void*)(&pCsr->aColumnSize[i]);
         1665  +          pCsr->aColumnSize[i] = 0;
         1666  +          rc = fts5ApiColumnText(pCtx, i, &z, &n);
         1667  +          if( rc==SQLITE_OK ){
         1668  +            rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb);
         1669  +          }
         1670  +        }
         1671  +      }
         1672  +    }
  1632   1673       CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
  1633   1674     }
  1634   1675     if( iCol<0 ){
  1635   1676       int i;
  1636   1677       *pnToken = 0;
  1637         -    for(i=0; i<pTab->pConfig->nCol; i++){
         1678  +    for(i=0; i<pConfig->nCol; i++){
  1638   1679         *pnToken += pCsr->aColumnSize[i];
  1639   1680       }
  1640         -  }else if( iCol<pTab->pConfig->nCol ){
         1681  +  }else if( iCol<pConfig->nCol ){
  1641   1682       *pnToken = pCsr->aColumnSize[iCol];
  1642   1683     }else{
  1643   1684       *pnToken = 0;
  1644   1685       rc = SQLITE_RANGE;
  1645   1686     }
  1646   1687     return rc;
  1647   1688   }
................................................................................
  1952   1993     }
  1953   1994   
  1954   1995     /* No function of the specified name was found. Return 0. */
  1955   1996     return 0;
  1956   1997   }
  1957   1998   
  1958   1999   /*
  1959         -** Implementation of FTS3 xRename method. Rename an fts5 table.
         2000  +** Implementation of FTS5 xRename method. Rename an fts5 table.
  1960   2001   */
  1961   2002   static int fts5RenameMethod(
  1962   2003     sqlite3_vtab *pVtab,            /* Virtual table handle */
  1963   2004     const char *zName               /* New name of table */
  1964   2005   ){
  1965   2006     int rc = SQLITE_OK;
  1966   2007     return rc;

Changes to ext/fts5/fts5Int.h.

   101    101   ** zContentExprlist:
   102    102   **
   103    103   ** pzErrmsg:
   104    104   **   This exists in order to allow the fts5_index.c module to return a 
   105    105   **   decent error message if it encounters a file-format version it does
   106    106   **   not understand.
   107    107   **
          108  +** bColumnsize:
          109  +**   True if the %_docsize table is created.
          110  +**
   108    111   */
   109    112   struct Fts5Config {
   110    113     sqlite3 *db;                    /* Database handle */
   111    114     char *zDb;                      /* Database holding FTS index (e.g. "main") */
   112    115     char *zName;                    /* Name of FTS index */
   113    116     int nCol;                       /* Number of columns */
   114    117     char **azCol;                   /* Column names */
   115    118     u8 *abUnindexed;                /* True for unindexed columns */
   116    119     int nPrefix;                    /* Number of prefix indexes */
   117    120     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   118    121     int eContent;                   /* An FTS5_CONTENT value */
   119    122     char *zContent;                 /* content table */ 
   120    123     char *zContentRowid;            /* "content_rowid=" option value */ 
          124  +  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
   121    125     char *zContentExprlist;
   122    126     Fts5Tokenizer *pTok;
   123    127     fts5_tokenizer *pTokApi;
   124    128   
   125    129     /* Values loaded from the %_config table */
   126    130     int iCookie;                    /* Incremented when %_config is modified */
   127    131     int pgsz;                       /* Approximate page size used in %_data */
................................................................................
   191    195   void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
   192    196   void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
   193    197   void sqlite3Fts5BufferFree(Fts5Buffer*);
   194    198   void sqlite3Fts5BufferZero(Fts5Buffer*);
   195    199   void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
   196    200   void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
   197    201   void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);
          202  +
          203  +char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...);
   198    204   
   199    205   #define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
   200    206   #define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
   201    207   #define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
   202    208   #define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
   203    209   #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
   204    210   #define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)

Changes to ext/fts5/fts5_buffer.c.

   120    120         *pRc = SQLITE_NOMEM;
   121    121       }else{
   122    122         sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
   123    123         sqlite3_free(zTmp);
   124    124       }
   125    125     }
   126    126   }
          127  +
          128  +char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...){
          129  +  char *zRet = 0;
          130  +  if( *pRc==SQLITE_OK ){
          131  +    va_list ap;
          132  +    va_start(ap, zFmt);
          133  +    zRet = sqlite3_vmprintf(zFmt, ap);
          134  +    va_end(ap);
          135  +    if( zRet==0 ){
          136  +      *pRc = SQLITE_NOMEM; 
          137  +    }
          138  +  }
          139  +  return zRet;
          140  +}
          141  + 
   127    142   
   128    143   /*
   129    144   ** Free any buffer allocated by pBuf. Zero the structure before returning.
   130    145   */
   131    146   void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
   132    147     sqlite3_free(pBuf->p);
   133    148     memset(pBuf, 0, sizeof(Fts5Buffer));

Changes to ext/fts5/fts5_config.c.

   192    192     quote = z[0];
   193    193     if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
   194    194       fts5Dequote(z);
   195    195     }
   196    196   }
   197    197   
   198    198   /*
   199         -** Parse the "special" CREATE VIRTUAL TABLE directive and update
          199  +** Parse a "special" CREATE VIRTUAL TABLE directive and update
   200    200   ** configuration object pConfig as appropriate.
   201    201   **
   202    202   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   203    203   ** an error occurs, an SQLite error code is returned and an error message
   204    204   ** may be left in *pzErr. It is the responsibility of the caller to
   205    205   ** eventually free any such error message using sqlite3_free().
   206    206   */
................................................................................
   207    207   static int fts5ConfigParseSpecial(
   208    208     Fts5Global *pGlobal,
   209    209     Fts5Config *pConfig,            /* Configuration object to update */
   210    210     const char *zCmd,               /* Special command to parse */
   211    211     const char *zArg,               /* Argument to parse */
   212    212     char **pzErr                    /* OUT: Error message */
   213    213   ){
          214  +  int rc = SQLITE_OK;
   214    215     int nCmd = strlen(zCmd);
   215    216     if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
   216    217       const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
   217         -    int rc = SQLITE_OK;
   218    218       const char *p;
   219    219       if( pConfig->aPrefix ){
   220    220         *pzErr = sqlite3_mprintf("multiple prefix=... directives");
   221    221         rc = SQLITE_ERROR;
   222    222       }else{
   223    223         pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
   224    224       }
................................................................................
   244    244         pConfig->aPrefix[pConfig->nPrefix] = nPre;
   245    245         pConfig->nPrefix++;
   246    246       }
   247    247       return rc;
   248    248     }
   249    249   
   250    250     if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
   251         -    int rc = SQLITE_OK;
   252    251       const char *p = (const char*)zArg;
   253    252       int nArg = strlen(zArg) + 1;
   254    253       char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
   255    254       char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
   256    255       char *pSpace = pDel;
   257    256   
   258    257       if( azArg && pSpace ){
................................................................................
   289    288   
   290    289       sqlite3_free(azArg);
   291    290       sqlite3_free(pDel);
   292    291       return rc;
   293    292     }
   294    293   
   295    294     if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
   296         -    int rc = SQLITE_OK;
   297    295       if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
   298    296         *pzErr = sqlite3_mprintf("multiple content=... directives");
   299    297         rc = SQLITE_ERROR;
   300    298       }else{
   301    299         if( zArg[0] ){
   302    300           pConfig->eContent = FTS5_CONTENT_EXTERNAL;
   303    301           pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
          302  +        if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
   304    303         }else{
   305    304           pConfig->eContent = FTS5_CONTENT_NONE;
   306         -        pConfig->zContent = sqlite3_mprintf(
   307         -            "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName
   308         -        );
   309    305         }
   310         -      if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
   311    306       }
   312    307       return rc;
   313    308     }
   314    309   
   315    310     if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
   316         -    int rc = SQLITE_OK;
   317    311       if( pConfig->zContentRowid ){
   318    312         *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
   319    313         rc = SQLITE_ERROR;
   320    314       }else{
   321    315         pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
   322    316       }
   323    317       return rc;
   324    318     }
          319  +
          320  +  if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
          321  +    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
          322  +      *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
          323  +      rc = SQLITE_ERROR;
          324  +    }else{
          325  +      pConfig->bColumnsize = (zArg[0]=='1');
          326  +    }
          327  +    return rc;
          328  +  }
   325    329   
   326    330     *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   327    331     return SQLITE_ERROR;
   328    332   }
   329    333   
   330    334   /*
   331    335   ** Allocate an instance of the default tokenizer ("simple") at 
................................................................................
   473    477     pRet->iCookie = -1;
   474    478   
   475    479     nByte = nArg * (sizeof(char*) + sizeof(u8));
   476    480     pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
   477    481     pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
   478    482     pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
   479    483     pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
          484  +  pRet->bColumnsize = 1;
   480    485     if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
   481    486       *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
   482    487       rc = SQLITE_ERROR;
   483    488     }
   484    489   
   485    490     for(i=3; rc==SQLITE_OK && i<nArg; i++){
   486    491       const char *zOrig = azArg[i];
................................................................................
   526    531     ** already been allocated. Otherwise, allocate an instance of the default
   527    532     ** tokenizer (unicode61) now.  */
   528    533     if( rc==SQLITE_OK && pRet->pTok==0 ){
   529    534       rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   530    535     }
   531    536   
   532    537     /* If no zContent option was specified, fill in the default values. */
   533         -  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
   534         -    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
   535         -    if( pRet->zContent==0 ){
   536         -      rc = SQLITE_NOMEM;
   537         -    }else{
   538         -      sqlite3_free(pRet->zContentRowid);
   539         -      pRet->zContentRowid = 0;
          538  +  if( rc==SQLITE_OK && pRet->zContent==0 ){
          539  +    const char *zTail = 0;
          540  +    assert( pRet->eContent==FTS5_CONTENT_NORMAL 
          541  +         || pRet->eContent==FTS5_CONTENT_NONE 
          542  +    );
          543  +    if( pRet->eContent==FTS5_CONTENT_NORMAL ){
          544  +      zTail = "content";
          545  +    }else if( pRet->bColumnsize ){
          546  +      zTail = "docsize";
          547  +    }
          548  +
          549  +    if( zTail ){
          550  +      pRet->zContent = sqlite3Fts5Mprintf(
          551  +          &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
          552  +      );
   540    553       }
   541    554     }
          555  +
   542    556     if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
   543    557       pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
   544    558     }
   545    559   
   546    560     /* Formulate the zContentExprlist text */
   547    561     if( rc==SQLITE_OK ){
   548    562       rc = fts5ConfigMakeExprlist(pRet);

Changes to ext/fts5/fts5_storage.c.

    35     35   #endif
    36     36   #if FTS5_STMT_LOOKUP!=2
    37     37   # error "FTS5_STMT_LOOKUP mismatch" 
    38     38   #endif
    39     39   
    40     40   #define FTS5_STMT_INSERT_CONTENT  3
    41     41   #define FTS5_STMT_REPLACE_CONTENT 4
    42         -
    43     42   #define FTS5_STMT_DELETE_CONTENT  5
    44     43   #define FTS5_STMT_REPLACE_DOCSIZE  6
    45     44   #define FTS5_STMT_DELETE_DOCSIZE  7
    46         -
    47     45   #define FTS5_STMT_LOOKUP_DOCSIZE  8
    48         -
    49     46   #define FTS5_STMT_REPLACE_CONFIG 9
    50         -
    51     47   #define FTS5_STMT_SCAN 10
    52     48   
    53     49   /*
    54     50   ** Prepare the two insert statements - Fts5Storage.pInsertContent and
    55     51   ** Fts5Storage.pInsertDocsize - if they have not already been prepared.
    56     52   ** Return SQLITE_OK if successful, or an SQLite error code if an error
    57     53   ** occurs.
................................................................................
    59     55   static int fts5StorageGetStmt(
    60     56     Fts5Storage *p,                 /* Storage handle */
    61     57     int eStmt,                      /* FTS5_STMT_XXX constant */
    62     58     sqlite3_stmt **ppStmt,          /* OUT: Prepared statement handle */
    63     59     char **pzErrMsg                 /* OUT: Error message (if any) */
    64     60   ){
    65     61     int rc = SQLITE_OK;
           62  +
           63  +  /* If there is no %_docsize table, there should be no requests for 
           64  +  ** statements to operate on it.  */
           65  +  assert( p->pConfig->bColumnsize || (
           66  +        eStmt!=FTS5_STMT_REPLACE_DOCSIZE 
           67  +     && eStmt!=FTS5_STMT_DELETE_DOCSIZE 
           68  +     && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE 
           69  +  ));
    66     70   
    67     71     assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
    68     72     if( p->aStmt[eStmt]==0 ){
    69     73       const char *azStmt[] = {
    70     74         "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
    71     75         "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
    72     76         "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
................................................................................
   171    175   /*
   172    176   ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
   173    177   ** code otherwise.
   174    178   */
   175    179   int sqlite3Fts5DropAll(Fts5Config *pConfig){
   176    180     int rc = fts5ExecPrintf(pConfig->db, 0, 
   177    181         "DROP TABLE IF EXISTS %Q.'%q_data';"
   178         -      "DROP TABLE IF EXISTS %Q.'%q_docsize';"
   179    182         "DROP TABLE IF EXISTS %Q.'%q_config';",
   180         -      pConfig->zDb, pConfig->zName,
   181    183         pConfig->zDb, pConfig->zName,
   182    184         pConfig->zDb, pConfig->zName
   183    185     );
          186  +  if( rc==SQLITE_OK && pConfig->bColumnsize ){
          187  +    rc = fts5ExecPrintf(pConfig->db, 0, 
          188  +        "DROP TABLE IF EXISTS %Q.'%q_docsize';",
          189  +        pConfig->zDb, pConfig->zName
          190  +    );
          191  +  }
   184    192     if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
   185    193       rc = fts5ExecPrintf(pConfig->db, 0, 
   186    194           "DROP TABLE IF EXISTS %Q.'%q_content';",
   187    195           pConfig->zDb, pConfig->zName
   188    196       );
   189    197     }
   190    198     return rc;
................................................................................
   262    270             iOff += strlen(&zDefn[iOff]);
   263    271           }
   264    272           rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
   265    273         }
   266    274         sqlite3_free(zDefn);
   267    275       }
   268    276   
   269         -    if( rc==SQLITE_OK ){
          277  +    if( rc==SQLITE_OK && pConfig->bColumnsize ){
   270    278         rc = sqlite3Fts5CreateTable(
   271    279             pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
   272    280         );
   273    281       }
   274    282       if( rc==SQLITE_OK ){
   275    283         rc = sqlite3Fts5CreateTable(
   276    284             pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
................................................................................
   370    378   }
   371    379   
   372    380   
   373    381   /*
   374    382   ** Insert a record into the %_docsize table. Specifically, do:
   375    383   **
   376    384   **   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
          385  +**
          386  +** If there is no %_docsize table (as happens if the columnsize=0 option
          387  +** is specified when the FTS5 table is created), this function is a no-op.
   377    388   */
   378    389   static int fts5StorageInsertDocsize(
   379    390     Fts5Storage *p,                 /* Storage module to write to */
   380    391     i64 iRowid,                     /* id value */
   381    392     Fts5Buffer *pBuf                /* sz value */
   382    393   ){
   383         -  sqlite3_stmt *pReplace = 0;
   384         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   385         -  if( rc==SQLITE_OK ){
   386         -    sqlite3_bind_int64(pReplace, 1, iRowid);
   387         -    sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
   388         -    sqlite3_step(pReplace);
   389         -    rc = sqlite3_reset(pReplace);
          394  +  int rc = SQLITE_OK;
          395  +  if( p->pConfig->bColumnsize ){
          396  +    sqlite3_stmt *pReplace = 0;
          397  +    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
          398  +    if( rc==SQLITE_OK ){
          399  +      sqlite3_bind_int64(pReplace, 1, iRowid);
          400  +      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
          401  +      sqlite3_step(pReplace);
          402  +      rc = sqlite3_reset(pReplace);
          403  +    }
   390    404     }
   391    405     return rc;
   392    406   }
   393    407   
   394    408   /*
   395    409   ** Load the contents of the "averages" record from disk into the 
   396    410   ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
................................................................................
   451    465     return rc;
   452    466   }
   453    467   
   454    468   /*
   455    469   ** Remove a row from the FTS table.
   456    470   */
   457    471   int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
          472  +  Fts5Config *pConfig = p->pConfig;
   458    473     int rc;
   459    474     sqlite3_stmt *pDel;
   460    475   
   461    476     rc = fts5StorageLoadTotals(p, 1);
   462    477   
   463    478     /* Delete the index records */
   464    479     if( rc==SQLITE_OK ){
   465    480       rc = fts5StorageDeleteFromIndex(p, iDel);
   466    481     }
   467    482   
   468    483     /* Delete the %_docsize record */
   469         -  if( rc==SQLITE_OK ){
          484  +  if( rc==SQLITE_OK && pConfig->bColumnsize ){
   470    485       rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   471    486     }
   472    487     if( rc==SQLITE_OK ){
   473    488       sqlite3_bind_int64(pDel, 1, iDel);
   474    489       sqlite3_step(pDel);
   475    490       rc = sqlite3_reset(pDel);
   476    491     }
................................................................................
   524    539         );
   525    540         p->aTotalSize[iCol] -= (i64)ctx.szCol;
   526    541       }
   527    542       p->nTotalRow--;
   528    543     }
   529    544   
   530    545     /* Delete the %_docsize record */
   531         -  if( rc==SQLITE_OK ){
   532         -    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
   533         -  }
   534         -  if( rc==SQLITE_OK ){
   535         -    sqlite3_bind_int64(pDel, 1, iDel);
   536         -    sqlite3_step(pDel);
   537         -    rc = sqlite3_reset(pDel);
          546  +  if( pConfig->bColumnsize ){
          547  +    if( rc==SQLITE_OK ){
          548  +      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
          549  +    }
          550  +    if( rc==SQLITE_OK ){
          551  +      sqlite3_bind_int64(pDel, 1, iDel);
          552  +      sqlite3_step(pDel);
          553  +      rc = sqlite3_reset(pDel);
          554  +    }
   538    555     }
   539    556   
   540    557     /* Write the averages record */
   541    558     if( rc==SQLITE_OK ){
   542    559       rc = fts5StorageSaveTotals(p);
   543    560     }
   544    561   
................................................................................
   550    567   */
   551    568   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
   552    569     Fts5Config *pConfig = p->pConfig;
   553    570     int rc;
   554    571   
   555    572     /* Delete the contents of the %_data and %_docsize tables. */
   556    573     rc = fts5ExecPrintf(pConfig->db, 0,
   557         -      "DELETE FROM %Q.'%q_data';"
   558         -      "DELETE FROM %Q.'%q_docsize';",
   559         -      pConfig->zDb, pConfig->zName,
          574  +      "DELETE FROM %Q.'%q_data';",
   560    575         pConfig->zDb, pConfig->zName
   561    576     );
          577  +  if( rc==SQLITE_OK && pConfig->bColumnsize ){
          578  +    rc = fts5ExecPrintf(pConfig->db, 0,
          579  +        "DELETE FROM %Q.'%q_docsize';",
          580  +        pConfig->zDb, pConfig->zName
          581  +    );
          582  +  }
   562    583   
   563    584     /* Reinitialize the %_data table. This call creates the initial structure
   564    585     ** and averages records.  */
   565    586     if( rc==SQLITE_OK ){
   566    587       rc = sqlite3Fts5IndexReinit(p->pIndex);
   567    588     }
   568    589     if( rc==SQLITE_OK ){
................................................................................
   631    652   }
   632    653   
   633    654   /*
   634    655   ** Allocate a new rowid. This is used for "external content" tables when
   635    656   ** a NULL value is inserted into the rowid column. The new rowid is allocated
   636    657   ** by inserting a dummy row into the %_docsize table. The dummy will be
   637    658   ** overwritten later.
          659  +**
          660  +** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
          661  +** this case the user is required to provide a rowid explicitly.
   638    662   */
   639    663   static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
   640         -  sqlite3_stmt *pReplace = 0;
   641         -  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
   642         -  if( rc==SQLITE_OK ){
   643         -    sqlite3_bind_null(pReplace, 1);
   644         -    sqlite3_bind_null(pReplace, 2);
   645         -    sqlite3_step(pReplace);
   646         -    rc = sqlite3_reset(pReplace);
   647         -  }
   648         -  if( rc==SQLITE_OK ){
   649         -    *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
          664  +  int rc = SQLITE_MISMATCH;
          665  +  if( p->pConfig->bColumnsize ){
          666  +    sqlite3_stmt *pReplace = 0;
          667  +    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
          668  +    if( rc==SQLITE_OK ){
          669  +      sqlite3_bind_null(pReplace, 1);
          670  +      sqlite3_bind_null(pReplace, 2);
          671  +      sqlite3_step(pReplace);
          672  +      rc = sqlite3_reset(pReplace);
          673  +    }
          674  +    if( rc==SQLITE_OK ){
          675  +      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
          676  +    }
   650    677     }
   651    678     return rc;
   652    679   }
   653    680   
   654    681   /*
   655    682   ** Insert a new row into the FTS table.
   656    683   */
................................................................................
   954    981         }
   955    982       }
   956    983       rc = sqlite3_reset(pLookup);
   957    984       if( bCorrupt && rc==SQLITE_OK ){
   958    985         rc = FTS5_CORRUPT;
   959    986       }
   960    987     }
          988  +
   961    989     return rc;
   962    990   }
   963    991   
   964    992   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
   965    993     int rc = fts5StorageLoadTotals(p, 0);
   966    994     if( rc==SQLITE_OK ){
   967    995       *pnToken = 0;

Changes to ext/fts5/test/fts5ae.test.

   144    144     ORDER BY rowid DESC;
   145    145   } {
   146    146     3 {1 0}
   147    147     2 {0 1}
   148    148     1 {4 6}
   149    149   }
   150    150   
   151         -do_execsql_test 5.2 {
          151  +do_execsql_test 5.3 {
   152    152     SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
   153    153     ORDER BY rowid DESC;
   154    154   } {
   155    155     3 {a {}}
   156    156     2 {{} a}
   157    157     1 {{a b c d} {e f g h i j}}
   158    158   }
   159    159   
   160         -do_execsql_test 5.3 {
          160  +do_execsql_test 5.4 {
   161    161     SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   162    162     ORDER BY rowid DESC;
   163    163   } {
   164    164     3 {5 7}
   165    165     2 {5 7}
   166    166     1 {5 7}
   167    167   }
   168    168   
   169         -do_execsql_test 5.4 {
          169  +do_execsql_test 5.5 {
   170    170     INSERT INTO t5 VALUES('x y z', 'v w x y z');
   171    171     SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
   172    172     ORDER BY rowid DESC;
   173    173   } {
   174    174     3 {8 12}
   175    175     2 {8 12}
   176    176     1 {8 12}

Added ext/fts5/test/fts5columnsize.test.

            1  +# 2015 Jun 10
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# Tests focusing on fts5 tables with the columnsize=0 option.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5columnsize
           17  +
           18  +#-------------------------------------------------------------------------
           19  +# Check that the option can be parsed and that the %_docsize table is
           20  +# only created if it is set to true.
           21  +#
           22  +foreach {tn outcome stmt} {
           23  +  1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) }
           24  +  2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) }
           25  +  3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') }
           26  +  4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') }
           27  +  5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') }
           28  +  6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) }
           29  +  7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) }
           30  +  8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) }
           31  +} {
           32  +  execsql { 
           33  +    DROP TABLE IF EXISTS t1;
           34  +  }
           35  +  if {$outcome==2} {
           36  +    do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}}
           37  +  } else {
           38  +    do_execsql_test 1.$tn.2 $stmt
           39  +    do_execsql_test 1.$tn.3 {
           40  +      SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize'
           41  +    } $outcome
           42  +  }
           43  +}
           44  +
           45  +#-------------------------------------------------------------------------
           46  +# Run tests on a table with no %_content or %_docsize backing store.
           47  +#
           48  +do_execsql_test 2.0 {
           49  +  CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content='');
           50  +}
           51  +do_catchsql_test 2.1 {
           52  +  INSERT INTO t2 VALUES('a b c d e f');
           53  +} {1 {datatype mismatch}}
           54  +do_execsql_test 2.2 {
           55  +  INSERT INTO t2(rowid, x) VALUES(1, 'c d e f');
           56  +  INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h');
           57  +  INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h');
           58  +} {}
           59  +do_execsql_test 2.3 {
           60  +  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
           61  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
           62  +  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
           63  +} {3 :: 1 2 3 :: 2 3}
           64  +do_execsql_test 2.4 {
           65  +  INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h');
           66  +  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
           67  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
           68  +  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
           69  +} {3 :: 1 3 :: 3}
           70  +do_execsql_test 2.5 {
           71  +  INSERT INTO t2(t2) VALUES('delete-all');
           72  +  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
           73  +  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
           74  +  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
           75  +} {:: ::}
           76  +do_execsql_test 2.6 {
           77  +  INSERT INTO t2(rowid, x) VALUES(1, 'o t t f');
           78  +  INSERT INTO t2(rowid, x) VALUES(2, 'f s s e');
           79  +  INSERT INTO t2(rowid, x) VALUES(3, 'n t e t');
           80  +}
           81  +
           82  +do_catchsql_test 2.7.1 {
           83  +  SELECT rowid FROM t2
           84  +} {1 {t2: table does not support scanning}}
           85  +do_catchsql_test 2.7.2 {
           86  +  SELECT rowid FROM t2 WHERE rowid=2
           87  +} {1 {t2: table does not support scanning}}
           88  +do_catchsql_test 2.7.3 {
           89  +  SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3
           90  +} {1 {t2: table does not support scanning}}
           91  +
           92  +do_execsql_test 2.X {
           93  +  DROP TABLE t2
           94  +}
           95  +
           96  +#-------------------------------------------------------------------------
           97  +# Test the xColumnSize() API
           98  +#
           99  +fts5_aux_test_functions db
          100  +
          101  +do_execsql_test 3.0 {
          102  +  CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0);
          103  +  INSERT INTO t3 VALUES('a a', 'b b b', 'c');
          104  +  INSERT INTO t3 VALUES('x a x', 'b b b y', '');
          105  +}
          106  +do_execsql_test 3.1 {
          107  +  SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a'
          108  +} {
          109  +  1 {2 0 1} 2 {3 0 0}
          110  +}
          111  +
          112  +finish_test