SQLite

Check-in [aa12f9d9b7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3".
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: aa12f9d9b79c2f523fd6b00e47bcb66dba09ce0c
User & Date: dan 2015-06-09 20:58:39.182
Context
2015-06-10
10:45
Fix the fts5 xRename() method. (check-in: 0f7fd51325 user: dan tags: fts5)
2015-06-09
20:58
Add the "columnsize=" option to fts5, similar to fts4's "matchinfo=fts3". (check-in: aa12f9d9b7 user: dan tags: fts5)
2015-06-06
19:23
Fix a comment in fts5.h. (check-in: e964b58774 user: dan tags: fts5)
Changes
Side-by-Side Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5.c.
1035
1036
1037
1038
1039
1040
1041

1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052

1053
1054
1055
1056
1057
1058
1059
1060
1061
1062

1063
1064
1065
1066
1067
1068
1069
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052

1053
1054
1055
1056
1057
1058
1059
1060
1061
1062

1063
1064
1065
1066
1067
1068
1069
1070







+










-
+









-
+







  sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  int idxNum,                     /* Strategy index */
  const char *idxStr,             /* Unused */
  int nVal,                       /* Number of elements in apVal */
  sqlite3_value **apVal           /* Arguments for the indexing scheme */
){
  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
  Fts5Config *pConfig = pTab->pConfig;
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;             /* Error code */
  int iVal = 0;                   /* Counter for apVal[] */
  int bDesc;                      /* True if ORDER BY [rank|rowid] DESC */
  int bOrderByRank;               /* True if ORDER BY rank */
  sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  char **pzErrmsg = pTab->pConfig->pzErrmsg;
  char **pzErrmsg = pConfig->pzErrmsg;

  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );
  assert( pCsr->zRank==0 );
  assert( pCsr->zRankArgs==0 );

  assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg );
  pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg;
  pConfig->pzErrmsg = &pTab->base.zErrMsg;

  /* Decode the arguments passed through to this function.
  **
  ** Note: The following set of if(...) statements must be in the same
  ** order as the corresponding entries in the struct at the top of
  ** fts5BestIndexMethod().  */
  if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
1103
1104
1105
1106
1107
1108
1109
1110

1111
1112
1113
1114
1115
1116
1117
1118
1119

1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130





1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149

1150
1151
1152
1153
1154
1155
1156
1104
1105
1106
1107
1108
1109
1110

1111
1112
1113
1114
1115
1116
1117
1118
1119

1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154

1155
1156
1157
1158
1159
1160
1161
1162







-
+








-
+











+
+
+
+
+


















-
+







    assert( pCsr->iFirstRowid==SMALLEST_INT64 );
    pCsr->ePlan = FTS5_PLAN_SOURCE;
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
  }else if( pMatch ){
    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);

    rc = fts5CursorParseRank(pTab->pConfig, pCsr, pRank);
    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
    if( rc==SQLITE_OK ){
      if( zExpr[0]=='*' ){
        /* The user has issued a query of the form "MATCH '*...'". This
        ** indicates that the MATCH expression is not a full text query,
        ** but a request for an internal parameter.  */
        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
      }else{
        char **pzErr = &pTab->base.zErrMsg;
        rc = sqlite3Fts5ExprNew(pTab->pConfig, zExpr, &pCsr->pExpr, pzErr);
        rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr);
        if( rc==SQLITE_OK ){
          if( bOrderByRank ){
            pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
            rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
          }else{
            pCsr->ePlan = FTS5_PLAN_MATCH;
            rc = fts5CursorFirst(pTab, pCsr, bDesc);
          }
        }
      }
    }
  }else if( pConfig->zContent==0 ){
    *pConfig->pzErrmsg = sqlite3_mprintf(
        "%s: table does not support scanning", pConfig->zName
    );
    rc = SQLITE_ERROR;
  }else{
    /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
    ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
    pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN);
    rc = sqlite3Fts5StorageStmt(
        pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg
    );
    if( rc==SQLITE_OK ){
      if( pCsr->ePlan==FTS5_PLAN_ROWID ){
        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
      }else{
        sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
        sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
      }
      rc = fts5NextMethod(pCursor);
    }
  }

  pTab->pConfig->pzErrmsg = pzErrmsg;
  pConfig->pzErrmsg = pzErrmsg;
  return rc;
}

/* 
** This is the xEof method of the virtual table. SQLite calls this 
** routine to find out if it has reached the end of a result set.
*/
1616
1617
1618
1619
1620
1621
1622












1623
1624
1625
1626

1627
1628
1629

1630
1631























1632
1633
1634
1635
1636
1637

1638
1639
1640

1641
1642
1643
1644
1645
1646
1647
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649


1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677

1678
1679
1680

1681
1682
1683
1684
1685
1686
1687
1688







+
+
+
+
+
+
+
+
+
+
+
+




+



+
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+





-
+


-
+







    if( rc==SQLITE_OK ){
      *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
      *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
    }
  }
  return rc;
}

static int fts5ColumnSizeCb(
  void *pContext,                 /* Pointer to int */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd                        /* End offset of token */
){
  int *pCnt = (int*)pContext;
  *pCnt = *pCnt + 1;
  return SQLITE_OK;
}

static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  Fts5Config *pConfig = pTab->pConfig;
  int rc = SQLITE_OK;

  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
    if( pConfig->bColumnsize ){
    i64 iRowid = fts5CursorRowid(pCsr);
    rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
      i64 iRowid = fts5CursorRowid(pCsr);
      rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
    }else if( pConfig->zContent==0 ){
      int i;
      for(i=0; i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i]==0 ){
          pCsr->aColumnSize[i] = -1;
        }
      }
    }else{
      int i;
      for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
        if( pConfig->abUnindexed[i]==0 ){
          const char *z; int n;
          void *p = (void*)(&pCsr->aColumnSize[i]);
          pCsr->aColumnSize[i] = 0;
          rc = fts5ApiColumnText(pCtx, i, &z, &n);
          if( rc==SQLITE_OK ){
            rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb);
          }
        }
      }
    }
    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
  }
  if( iCol<0 ){
    int i;
    *pnToken = 0;
    for(i=0; i<pTab->pConfig->nCol; i++){
    for(i=0; i<pConfig->nCol; i++){
      *pnToken += pCsr->aColumnSize[i];
    }
  }else if( iCol<pTab->pConfig->nCol ){
  }else if( iCol<pConfig->nCol ){
    *pnToken = pCsr->aColumnSize[iCol];
  }else{
    *pnToken = 0;
    rc = SQLITE_RANGE;
  }
  return rc;
}
1952
1953
1954
1955
1956
1957
1958
1959

1960
1961
1962
1963
1964
1965
1966
1993
1994
1995
1996
1997
1998
1999

2000
2001
2002
2003
2004
2005
2006
2007







-
+







  }

  /* No function of the specified name was found. Return 0. */
  return 0;
}

/*
** Implementation of FTS3 xRename method. Rename an fts5 table.
** Implementation of FTS5 xRename method. Rename an fts5 table.
*/
static int fts5RenameMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  const char *zName               /* New name of table */
){
  int rc = SQLITE_OK;
  return rc;
Changes to ext/fts5/fts5Int.h.
101
102
103
104
105
106
107



108
109
110
111
112
113
114
115
116
117
118
119
120

121
122
123
124
125
126
127
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131







+
+
+













+







** zContentExprlist:
**
** pzErrmsg:
**   This exists in order to allow the fts5_index.c module to return a 
**   decent error message if it encounters a file-format version it does
**   not understand.
**
** bColumnsize:
**   True if the %_docsize table is created.
**
*/
struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  u8 *abUnindexed;                /* True for unindexed columns */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
  char *zContentExprlist;
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */
191
192
193
194
195
196
197


198
199
200
201
202
203
204
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210







+
+







void sqlite3Fts5BufferAppendBlob(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendString(int *, Fts5Buffer*, const char*);
void sqlite3Fts5BufferFree(Fts5Buffer*);
void sqlite3Fts5BufferZero(Fts5Buffer*);
void sqlite3Fts5BufferSet(int*, Fts5Buffer*, int, const u8*);
void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...);
void sqlite3Fts5BufferAppend32(int*, Fts5Buffer*, int);

char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...);

#define fts5BufferZero(x)             sqlite3Fts5BufferZero(x)
#define fts5BufferGrow(a,b,c)         sqlite3Fts5BufferGrow(a,b,c)
#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c)
#define fts5BufferFree(a)             sqlite3Fts5BufferFree(a)
#define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d)
#define fts5BufferSet(a,b,c,d)        sqlite3Fts5BufferSet(a,b,c,d)
Changes to ext/fts5/fts5_buffer.c.
120
121
122
123
124
125
126















127
128
129
130
131
132
133
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







      *pRc = SQLITE_NOMEM;
    }else{
      sqlite3Fts5BufferAppendString(pRc, pBuf, zTmp);
      sqlite3_free(zTmp);
    }
  }
}

char *sqlite3Fts5Mprintf(int *pRc, char *zFmt, ...){
  char *zRet = 0;
  if( *pRc==SQLITE_OK ){
    va_list ap;
    va_start(ap, zFmt);
    zRet = sqlite3_vmprintf(zFmt, ap);
    va_end(ap);
    if( zRet==0 ){
      *pRc = SQLITE_NOMEM; 
    }
  }
  return zRet;
}
 

/*
** Free any buffer allocated by pBuf. Zero the structure before returning.
*/
void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){
  sqlite3_free(pBuf->p);
  memset(pBuf, 0, sizeof(Fts5Buffer));
Changes to ext/fts5/fts5_config.c.
192
193
194
195
196
197
198
199

200
201
202
203
204
205
206
207
208
209
210
211
212
213

214
215
216
217
218
219
220
221
222
223
224
192
193
194
195
196
197
198

199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217

218
219
220
221
222
223
224







-
+














+



-







  quote = z[0];
  if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
    fts5Dequote(z);
  }
}

/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
** may be left in *pzErr. It is the responsibility of the caller to
** eventually free any such error message using sqlite3_free().
*/
static int fts5ConfigParseSpecial(
  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */
  const char *zArg,               /* Argument to parse */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE_OK;
  int nCmd = strlen(zCmd);
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    int rc = SQLITE_OK;
    const char *p;
    if( pConfig->aPrefix ){
      *pzErr = sqlite3_mprintf("multiple prefix=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->aPrefix = sqlite3Fts5MallocZero(&rc, nByte);
    }
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
244
245
246
247
248
249
250

251
252
253
254
255
256
257







-







      pConfig->aPrefix[pConfig->nPrefix] = nPre;
      pConfig->nPrefix++;
    }
    return rc;
  }

  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    const char *p = (const char*)zArg;
    int nArg = strlen(zArg) + 1;
    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
    char *pSpace = pDel;

    if( azArg && pSpace ){
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303

304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324










325
326
327
328
329
330
331
288
289
290
291
292
293
294

295
296
297
298
299
300
301
302
303
304



305

306
307
308
309
310

311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335







-







+


-
-
-

-





-








+
+
+
+
+
+
+
+
+
+








    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
  }

  if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
      *pzErr = sqlite3_mprintf("multiple content=... directives");
      rc = SQLITE_ERROR;
    }else{
      if( zArg[0] ){
        pConfig->eContent = FTS5_CONTENT_EXTERNAL;
        pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
        if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
      }else{
        pConfig->eContent = FTS5_CONTENT_NONE;
        pConfig->zContent = sqlite3_mprintf(
            "%Q.'%q_docsize'", pConfig->zDb, pConfig->zName
        );
      }
      if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
    }
    return rc;
  }

  if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    if( pConfig->zContentRowid ){
      *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->zContentRowid = sqlite3Fts5Strndup(&rc, zArg, -1);
    }
    return rc;
  }

  if( sqlite3_strnicmp("columnsize", zCmd, nCmd)==0 ){
    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
      *pzErr = sqlite3_mprintf("malformed columnsize=... directive");
      rc = SQLITE_ERROR;
    }else{
      pConfig->bColumnsize = (zArg[0]=='1');
    }
    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
  return SQLITE_ERROR;
}

/*
** Allocate an instance of the default tokenizer ("simple") at 
473
474
475
476
477
478
479

480
481
482
483
484
485
486
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491







+







  pRet->iCookie = -1;

  nByte = nArg * (sizeof(char*) + sizeof(u8));
  pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte);
  pRet->abUnindexed = (u8*)&pRet->azCol[nArg];
  pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
  pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
  pRet->bColumnsize = 1;
  if( rc==SQLITE_OK && sqlite3_stricmp(pRet->zName, FTS5_RANK_NAME)==0 ){
    *pzErr = sqlite3_mprintf("reserved fts5 table name: %s", pRet->zName);
    rc = SQLITE_ERROR;
  }

  for(i=3; rc==SQLITE_OK && i<nArg; i++){
    const char *zOrig = azArg[i];
526
527
528
529
530
531
532
533
534
535
536
537
538
539















540
541

542
543
544
545
546
547
548
531
532
533
534
535
536
537







538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562







-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


+







  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (unicode61) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->eContent==FTS5_CONTENT_NORMAL ){
    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
    if( pRet->zContent==0 ){
      rc = SQLITE_NOMEM;
    }else{
      sqlite3_free(pRet->zContentRowid);
      pRet->zContentRowid = 0;
  if( rc==SQLITE_OK && pRet->zContent==0 ){
    const char *zTail = 0;
    assert( pRet->eContent==FTS5_CONTENT_NORMAL 
         || pRet->eContent==FTS5_CONTENT_NONE 
    );
    if( pRet->eContent==FTS5_CONTENT_NORMAL ){
      zTail = "content";
    }else if( pRet->bColumnsize ){
      zTail = "docsize";
    }

    if( zTail ){
      pRet->zContent = sqlite3Fts5Mprintf(
          &rc, "%Q.'%q_%s'", pRet->zDb, pRet->zName, zTail
      );
    }
  }

  if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
    pRet->zContentRowid = sqlite3Fts5Strndup(&rc, "rowid", -1);
  }

  /* Formulate the zContentExprlist text */
  if( rc==SQLITE_OK ){
    rc = fts5ConfigMakeExprlist(pRet);
Changes to ext/fts5/fts5_storage.c.
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65








66
67
68
69
70
71
72
35
36
37
38
39
40
41

42
43
44

45

46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76







-



-

-

-















+
+
+
+
+
+
+
+







#endif
#if FTS5_STMT_LOOKUP!=2
# error "FTS5_STMT_LOOKUP mismatch" 
#endif

#define FTS5_STMT_INSERT_CONTENT  3
#define FTS5_STMT_REPLACE_CONTENT 4

#define FTS5_STMT_DELETE_CONTENT  5
#define FTS5_STMT_REPLACE_DOCSIZE  6
#define FTS5_STMT_DELETE_DOCSIZE  7

#define FTS5_STMT_LOOKUP_DOCSIZE  8

#define FTS5_STMT_REPLACE_CONFIG 9

#define FTS5_STMT_SCAN 10

/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/
static int fts5StorageGetStmt(
  Fts5Storage *p,                 /* Storage handle */
  int eStmt,                      /* FTS5_STMT_XXX constant */
  sqlite3_stmt **ppStmt,          /* OUT: Prepared statement handle */
  char **pzErrMsg                 /* OUT: Error message (if any) */
){
  int rc = SQLITE_OK;

  /* If there is no %_docsize table, there should be no requests for 
  ** statements to operate on it.  */
  assert( p->pConfig->bColumnsize || (
        eStmt!=FTS5_STMT_REPLACE_DOCSIZE 
     && eStmt!=FTS5_STMT_DELETE_DOCSIZE 
     && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE 
  ));

  assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
  if( p->aStmt[eStmt]==0 ){
    const char *azStmt[] = {
      "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
      "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
171
172
173
174
175
176
177
178
179
180
181
182
183






184
185
186
187
188
189
190
175
176
177
178
179
180
181

182

183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198







-

-



+
+
+
+
+
+







/*
** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error
** code otherwise.
*/
int sqlite3Fts5DropAll(Fts5Config *pConfig){
  int rc = fts5ExecPrintf(pConfig->db, 0, 
      "DROP TABLE IF EXISTS %Q.'%q_data';"
      "DROP TABLE IF EXISTS %Q.'%q_docsize';"
      "DROP TABLE IF EXISTS %Q.'%q_config';",
      pConfig->zDb, pConfig->zName,
      pConfig->zDb, pConfig->zName,
      pConfig->zDb, pConfig->zName
  );
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5ExecPrintf(pConfig->db, 0, 
        "DROP TABLE IF EXISTS %Q.'%q_docsize';",
        pConfig->zDb, pConfig->zName
    );
  }
  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
    rc = fts5ExecPrintf(pConfig->db, 0, 
        "DROP TABLE IF EXISTS %Q.'%q_content';",
        pConfig->zDb, pConfig->zName
    );
  }
  return rc;
262
263
264
265
266
267
268
269

270
271
272
273
274
275
276
270
271
272
273
274
275
276

277
278
279
280
281
282
283
284







-
+







          iOff += strlen(&zDefn[iOff]);
        }
        rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
      }
      sqlite3_free(zDefn);
    }

    if( rc==SQLITE_OK ){
    if( rc==SQLITE_OK && pConfig->bColumnsize ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
      );
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
370
371
372
373
374
375
376



377
378
379
380
381
382


383
384
385
386
387
388
389








390
391
392
393
394
395
396
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395







396
397
398
399
400
401
402
403
404
405
406
407
408
409
410







+
+
+






+
+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+







}


/*
** Insert a record into the %_docsize table. Specifically, do:
**
**   INSERT OR REPLACE INTO %_docsize(id, sz) VALUES(iRowid, pBuf);
**
** If there is no %_docsize table (as happens if the columnsize=0 option
** is specified when the FTS5 table is created), this function is a no-op.
*/
static int fts5StorageInsertDocsize(
  Fts5Storage *p,                 /* Storage module to write to */
  i64 iRowid,                     /* id value */
  Fts5Buffer *pBuf                /* sz value */
){
  int rc = SQLITE_OK;
  if( p->pConfig->bColumnsize ){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pReplace, 1, iRowid);
    sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
    sqlite3_stmt *pReplace = 0;
    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pReplace, 1, iRowid);
      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
      sqlite3_step(pReplace);
      rc = sqlite3_reset(pReplace);
    }
  }
  return rc;
}

/*
** Load the contents of the "averages" record from disk into the 
** p->nTotalRow and p->aTotalSize[] variables. If successful, and if
451
452
453
454
455
456
457

458
459
460
461
462
463
464
465
466
467
468
469

470
471
472
473
474
475
476
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483

484
485
486
487
488
489
490
491







+











-
+







  return rc;
}

/*
** Remove a row from the FTS table.
*/
int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel;

  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    rc = fts5StorageDeleteFromIndex(p, iDel);
  }

  /* Delete the %_docsize record */
  if( rc==SQLITE_OK ){
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
  }
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pDel, 1, iDel);
    sqlite3_step(pDel);
    rc = sqlite3_reset(pDel);
  }
524
525
526
527
528
529
530

531
532
533
534
535
536
537








538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558






559
560
561



562
563
564
565
566
567
568
539
540
541
542
543
544
545
546







547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573


574
575
576
577
578
579



580
581
582
583
584
585
586
587
588
589







+
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+



















-
-
+
+
+
+
+
+
-
-
-
+
+
+







      );
      p->aTotalSize[iCol] -= (i64)ctx.szCol;
    }
    p->nTotalRow--;
  }

  /* Delete the %_docsize record */
  if( pConfig->bColumnsize ){
  if( rc==SQLITE_OK ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
  }
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pDel, 1, iDel);
    sqlite3_step(pDel);
    rc = sqlite3_reset(pDel);
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel, 0);
    }
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pDel, 1, iDel);
      sqlite3_step(pDel);
      rc = sqlite3_reset(pDel);
    }
  }

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}

/*
** Delete all entries in the FTS5 index.
*/
int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
  Fts5Config *pConfig = p->pConfig;
  int rc;

  /* Delete the contents of the %_data and %_docsize tables. */
  rc = fts5ExecPrintf(pConfig->db, 0,
      "DELETE FROM %Q.'%q_data';"
      "DELETE FROM %Q.'%q_docsize';",
      "DELETE FROM %Q.'%q_data';",
      pConfig->zDb, pConfig->zName
  );
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5ExecPrintf(pConfig->db, 0,
        "DELETE FROM %Q.'%q_docsize';",
      pConfig->zDb, pConfig->zName,
      pConfig->zDb, pConfig->zName
  );
        pConfig->zDb, pConfig->zName
    );
  }

  /* Reinitialize the %_data table. This call creates the initial structure
  ** and averages records.  */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexReinit(p->pIndex);
  }
  if( rc==SQLITE_OK ){
631
632
633
634
635
636
637



638
639


640
641
642
643
644
645
646
647
648
649











650
651
652
653
654
655
656
652
653
654
655
656
657
658
659
660
661
662
663
664
665










666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683







+
+
+


+
+
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+







}

/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.
**
** If the %_docsize table does not exist, SQLITE_MISMATCH is returned. In
** this case the user is required to provide a rowid explicitly.
*/
static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
  int rc = SQLITE_MISMATCH;
  if( p->pConfig->bColumnsize ){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_null(pReplace, 1);
    sqlite3_bind_null(pReplace, 2);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  if( rc==SQLITE_OK ){
    *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
    sqlite3_stmt *pReplace = 0;
    rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
    if( rc==SQLITE_OK ){
      sqlite3_bind_null(pReplace, 1);
      sqlite3_bind_null(pReplace, 2);
      sqlite3_step(pReplace);
      rc = sqlite3_reset(pReplace);
    }
    if( rc==SQLITE_OK ){
      *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
    }
  }
  return rc;
}

/*
** Insert a new row into the FTS table.
*/
954
955
956
957
958
959
960

961
962
963
964
965
966
967
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995







+







      }
    }
    rc = sqlite3_reset(pLookup);
    if( bCorrupt && rc==SQLITE_OK ){
      rc = FTS5_CORRUPT;
    }
  }

  return rc;
}

int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){
  int rc = fts5StorageLoadTotals(p, 0);
  if( rc==SQLITE_OK ){
    *pnToken = 0;
Changes to ext/fts5/test/fts5ae.test.
144
145
146
147
148
149
150
151

152
153
154
155
156
157
158
159
160

161
162
163
164
165
166
167
168
169

170
171
172
173
174
175
176
144
145
146
147
148
149
150

151
152
153
154
155
156
157
158
159

160
161
162
163
164
165
166
167
168

169
170
171
172
173
174
175
176







-
+








-
+








-
+







  ORDER BY rowid DESC;
} {
  3 {1 0}
  2 {0 1}
  1 {4 6}
}

do_execsql_test 5.2 {
do_execsql_test 5.3 {
  SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
do_execsql_test 5.4 {
  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {5 7}
  2 {5 7}
  1 {5 7}
}

do_execsql_test 5.4 {
do_execsql_test 5.5 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {8 12}
  2 {8 12}
  1 {8 12}
Added ext/fts5/test/fts5columnsize.test.
















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
# 2015 Jun 10
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on fts5 tables with the columnsize=0 option.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5columnsize

#-------------------------------------------------------------------------
# Check that the option can be parsed and that the %_docsize table is
# only created if it is set to true.
#
foreach {tn outcome stmt} {
  1 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0) }
  2 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=1) }
  3 0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='0') }
  4 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='1') }
  5 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize='') }
  6 2 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=2) }
  7 1 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0, columnsize=1) }
  8 1 { CREATE VIRTUAL TABLE t1 USING fts5(x) }
} {
  execsql { 
    DROP TABLE IF EXISTS t1;
  }
  if {$outcome==2} {
    do_catchsql_test 1.$tn.1 $stmt {1 {malformed columnsize=... directive}}
  } else {
    do_execsql_test 1.$tn.2 $stmt
    do_execsql_test 1.$tn.3 {
      SELECT count(*) FROM sqlite_master WHERE name = 't1_docsize'
    } $outcome
  }
}

#-------------------------------------------------------------------------
# Run tests on a table with no %_content or %_docsize backing store.
#
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, columnsize=0, content='');
}
do_catchsql_test 2.1 {
  INSERT INTO t2 VALUES('a b c d e f');
} {1 {datatype mismatch}}
do_execsql_test 2.2 {
  INSERT INTO t2(rowid, x) VALUES(1, 'c d e f');
  INSERT INTO t2(rowid, x) VALUES(2, 'c d e f g h');
  INSERT INTO t2(rowid, x) VALUES(3, 'a b c d e f g h');
} {}
do_execsql_test 2.3 {
  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
} {3 :: 1 2 3 :: 2 3}
do_execsql_test 2.4 {
  INSERT INTO t2(t2, rowid, x) VALUES('delete', 2, 'c d e f g h');
  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
} {3 :: 1 3 :: 3}
do_execsql_test 2.5 {
  INSERT INTO t2(t2) VALUES('delete-all');
  SELECT rowid FROM t2 WHERE t2 MATCH 'b'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'e'; SELECT '::';
  SELECT rowid FROM t2 WHERE t2 MATCH 'h'; 
} {:: ::}
do_execsql_test 2.6 {
  INSERT INTO t2(rowid, x) VALUES(1, 'o t t f');
  INSERT INTO t2(rowid, x) VALUES(2, 'f s s e');
  INSERT INTO t2(rowid, x) VALUES(3, 'n t e t');
}

do_catchsql_test 2.7.1 {
  SELECT rowid FROM t2
} {1 {t2: table does not support scanning}}
do_catchsql_test 2.7.2 {
  SELECT rowid FROM t2 WHERE rowid=2
} {1 {t2: table does not support scanning}}
do_catchsql_test 2.7.3 {
  SELECT rowid FROM t2 WHERE rowid BETWEEN 1 AND 3
} {1 {t2: table does not support scanning}}

do_execsql_test 2.X {
  DROP TABLE t2
}

#-------------------------------------------------------------------------
# Test the xColumnSize() API
#
fts5_aux_test_functions db

do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x, y UNINDEXED, z, columnsize=0);
  INSERT INTO t3 VALUES('a a', 'b b b', 'c');
  INSERT INTO t3 VALUES('x a x', 'b b b y', '');
}
do_execsql_test 3.1 {
  SELECT rowid, fts5_test_columnsize(t3) FROM t3 WHERE t3 MATCH 'a'
} {
  1 {2 0 1} 2 {3 0 0}
}

finish_test