SQLite

Check-in [17ef5b59f7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for external content tables to fts5.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 17ef5b59f789e9fa35c4f053246d819987fd06f8
User & Date: dan 2015-01-03 20:44:58.134
Context
2015-01-05
20:41
Tests and fixes for fts5 external content tables. (check-in: 047aaf830d user: dan tags: fts5)
2015-01-03
20:44
Add support for external content tables to fts5. (check-in: 17ef5b59f7 user: dan tags: fts5)
2015-01-02
14:55
Allow the rank column to be remapped on a per-query basis by including a term similar to "rank match 'bm25(10,2)'" in a where clause. (check-in: 1cd15a1759 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5.c.
1051
1052
1053
1054
1055
1056
1057














1058
1059
1060
1061
1062
1063
1064
      }else{
        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
      }
    }
  }
  return rc;
}















/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.
*/
static int fts5UpdateMethod(







>
>
>
>
>
>
>
>
>
>
>
>
>
>







1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
      }else{
        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
      }
    }
  }
  return rc;
}

static int fts5SpecialDelete(
  Fts5Table *pTab, 
  sqlite3_value **apVal, 
  sqlite3_int64 *piRowid
){
  int rc = SQLITE_OK;
  int eType1 = sqlite3_value_type(apVal[1]);
  if( eType1==SQLITE_INTEGER ){
    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
  }
  return rc;
}

/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.
*/
static int fts5UpdateMethod(
1082
1083
1084
1085
1086
1087
1088

1089



1090

1091
1092


1093
1094
1095
1096
1097
1098


1099
1100
1101
1102
1103
1104
1105
1106
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );


  if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){



    return fts5SpecialInsert(pTab, 

        apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
    );


  }

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );


  if( eType0==SQLITE_INTEGER ){
    i64 iDel = sqlite3_value_int64(apVal[0]);    /* Rowid to delete */
    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
  }

  if( rc==SQLITE_OK && nArg>1 ){
    rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  }







>
|
>
>
>
|
>
|
<
>
>






>
>
|







1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110

1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
  **   1. The "old" rowid, or NULL.
  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );

  if( nArg>1 ){
    sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]);
    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
      const char *z = sqlite3_value_text(pCmd);
      if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){
        return fts5SpecialDelete(pTab, apVal, pRowid);
      }else{
        return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);

      }
    }
  }

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  assert( pVtab->zErrMsg==0 );

  if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
    i64 iDel = sqlite3_value_int64(apVal[0]);    /* Rowid to delete */
    rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
  }

  if( rc==SQLITE_OK && nArg>1 ){
    rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  }
Changes to ext/fts5/fts5Int.h.
72
73
74
75
76
77
78



79
80
81
82
83
84
85
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */



  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */
  int nAutomerge;                 /* 'automerge' setting */







>
>
>







72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int bExternalContent;           /* Content is external */
  char *zContent;                 /* "content=" option value (or NULL) */ 
  char *zContentRowid;            /* "content_rowid=" option value (or NULL) */ 
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */
  int nAutomerge;                 /* 'automerge' setting */
405
406
407
408
409
410
411


412
413
414
415
416
417
418
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);



/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************







>
>







408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);

int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);

/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************
Changes to ext/fts5/fts5_config.c.
187
188
189
190
191
192
193









































194
195
196
197
198
199
200
  int n = strlen(z);
  while( n>0 && fts5_iswhitespace(z[n-1]) ){
    z[--n] = '\0';
  }
  while( fts5_iswhitespace(*z) ) z++;
  return z;
}










































/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
  int n = strlen(z);
  while( n>0 && fts5_iswhitespace(z[n-1]) ){
    z[--n] = '\0';
  }
  while( fts5_iswhitespace(*z) ) z++;
  return z;
}

/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){
  char *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_mprintf("%s", z);
    if( pRet==0 ) *pRc = SQLITE_NOMEM;
  }
  return pRet;
}

/*
** Argument z points to a nul-terminated string containing an SQL identifier.
** This function returns a copy of the identifier enclosed in backtick 
** quotes.
*/
static char *fts5EscapeName(int *pRc, const char *z){
  char *pRet = 0;
  if( *pRc==SQLITE_OK ){
    int n = strlen(z);
    pRet = (char*)sqlite3_malloc(2 * 2*n + 1);
    if( pRet==0 ){
      *pRc = SQLITE_NOMEM;
    }else{
      int i;
      char *p = pRet;
      for(i=0; i<n; i++){
        if( z[i]=='`' ) *p++ = '`';
        *p++ = z[i];
      }
      *p++ = '`';
      *p++ = '\0';
    }
  }
  return pRet;
}

/*
** Parse the "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
**
** If successful, object pConfig is updated and SQLITE_OK returned. If
** an error occurs, an SQLite error code is returned and an error message
286
287
288
289
290
291
292
























293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
  }

























  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
  return SQLITE_ERROR;
}

/*
** Duplicate the string passed as the only argument into a buffer allocated
** by sqlite3_malloc().
**
** Return 0 if an OOM error is encountered.
*/
static char *fts5Strdup(int *pRc, const char *z){
  char *pRet = 0;
  if( *pRc==SQLITE_OK ){
    pRet = sqlite3_mprintf("%s", z);
    if( pRet==0 ) *pRc = SQLITE_NOMEM;
  }
  return pRet;
}

/*
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





<
<
<
<
<
<
<
<
<
<
<
<
<
<
<







327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362















363
364
365
366
367
368
369
      }
    }

    sqlite3_free(azArg);
    sqlite3_free(pDel);
    return rc;
  }

  if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    if( pConfig->zContent ){
      *pzErr = sqlite3_mprintf("multiple content=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
      pConfig->bExternalContent = 1;
      if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
    }
    return rc;
  }

  if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
    int rc = SQLITE_OK;
    if( pConfig->zContentRowid ){
      *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
      rc = SQLITE_ERROR;
    }else{
      pConfig->zContentRowid = fts5EscapeName(&rc, zArg);
    }
    return rc;
  }

  *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
  return SQLITE_ERROR;
}
















/*
** Allocate an instance of the default tokenizer ("simple") at 
** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
** code if an error occurs.
*/
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
417
418
419
420
421
422
423














424
425
426
427
428
429
430

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (simple) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }















  if( rc!=SQLITE_OK ){
    sqlite3Fts5ConfigFree(pRet);
    *ppOut = 0;
  }
  return rc;
}







>
>
>
>
>
>
>
>
>
>
>
>
>
>







467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494

  /* If a tokenizer= option was successfully parsed, the tokenizer has
  ** already been allocated. Otherwise, allocate an instance of the default
  ** tokenizer (simple) now.  */
  if( rc==SQLITE_OK && pRet->pTok==0 ){
    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->zContent==0 ){
    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
    if( pRet->zContent==0 ){
      rc = SQLITE_NOMEM;
    }else{
      sqlite3_free(pRet->zContentRowid);
      pRet->zContentRowid = 0;
    }
  }
  if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
    pRet->zContentRowid = fts5Strdup(&rc, "rowid");
  }

  if( rc!=SQLITE_OK ){
    sqlite3Fts5ConfigFree(pRet);
    *ppOut = 0;
  }
  return rc;
}
443
444
445
446
447
448
449


450
451
452
453
454
455
456
    for(i=0; i<pConfig->nCol; i++){
      sqlite3_free(pConfig->azCol[i]);
    }
    sqlite3_free(pConfig->azCol);
    sqlite3_free(pConfig->aPrefix);
    sqlite3_free(pConfig->zRank);
    sqlite3_free(pConfig->zRankArgs);


    sqlite3_free(pConfig);
  }
}

/*
** Call sqlite3_declare_vtab() based on the contents of the configuration
** object passed as the only argument. Return SQLITE_OK if successful, or







>
>







507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
    for(i=0; i<pConfig->nCol; i++){
      sqlite3_free(pConfig->azCol[i]);
    }
    sqlite3_free(pConfig->azCol);
    sqlite3_free(pConfig->aPrefix);
    sqlite3_free(pConfig->zRank);
    sqlite3_free(pConfig->zRankArgs);
    sqlite3_free(pConfig->zContent);
    sqlite3_free(pConfig->zContentRowid);
    sqlite3_free(pConfig);
  }
}

/*
** Call sqlite3_declare_vtab() based on the contents of the configuration
** object passed as the only argument. Return SQLITE_OK if successful, or
Changes to ext/fts5/fts5_expr.c.
1037
1038
1039
1040
1041
1042
1043
1044


1045
1046
1047
1048



1049
1050
1051
1052
1053

1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
** Argument pIn points to a buffer of nIn bytes. This function allocates
** and returns a new buffer populated with a copy of (pIn/nIn) with a 
** nul-terminator byte appended to it.
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. 
*/
static char *fts5Strndup(const char *pIn, int nIn){


  char *zRet = (char*)sqlite3_malloc(nIn+1);
  if( zRet ){
    memcpy(zRet, pIn, nIn);
    zRet[nIn] = '\0';



  }
  return zRet;
}

static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){

  *pz = fts5Strndup(pToken->p, pToken->n);
  if( *pz==0 ) return SQLITE_NOMEM;
  return SQLITE_OK;
}

/*
** Free the phrase object passed as the only argument.
*/
static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
  if( pPhrase ){







|
>
>
|
|
|
|
>
>
>





>
|
<
|







1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060

1061
1062
1063
1064
1065
1066
1067
1068
** Argument pIn points to a buffer of nIn bytes. This function allocates
** and returns a new buffer populated with a copy of (pIn/nIn) with a 
** nul-terminator byte appended to it.
**
** It is the responsibility of the caller to eventually free the returned
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. 
*/
static char *fts5Strndup(int *pRc, const char *pIn, int nIn){
  char *zRet = 0;
  if( *pRc==SQLITE_OK ){
    zRet = (char*)sqlite3_malloc(nIn+1);
    if( zRet ){
      memcpy(zRet, pIn, nIn);
      zRet[nIn] = '\0';
    }else{
      *pRc = SQLITE_NOMEM;
    }
  }
  return zRet;
}

static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
  int rc = SQLITE_OK;
  *pz = fts5Strndup(&rc, pToken->p, pToken->n);

  return rc;
}

/*
** Free the phrase object passed as the only argument.
*/
static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
  if( pPhrase ){
1135
1136
1137
1138
1139
1140
1141

1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
  void *pContext,                 /* Pointer to Fts5InsertCtx object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){

  const int SZALLOC = 8;
  TokenCtx *pCtx = (TokenCtx*)pContext;
  Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
  Fts5ExprTerm *pTerm;

  if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
    Fts5ExprPhrase *pNew;
    int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);

    pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
        sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
    );
    if( pNew==0 ) return SQLITE_NOMEM;
    if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
    pCtx->pPhrase = pPhrase = pNew;
    pNew->nTerm = nNew - SZALLOC;
  }

  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  memset(pTerm, 0, sizeof(Fts5ExprTerm));
  pTerm->zTerm = fts5Strndup(pToken, nToken);

  return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
}


/*
** Free the phrase object passed as the only argument.
*/
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){







>




















|

|







1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
  void *pContext,                 /* Pointer to Fts5InsertCtx object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){
  int rc = SQLITE_OK;
  const int SZALLOC = 8;
  TokenCtx *pCtx = (TokenCtx*)pContext;
  Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
  Fts5ExprTerm *pTerm;

  if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
    Fts5ExprPhrase *pNew;
    int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);

    pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
        sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
    );
    if( pNew==0 ) return SQLITE_NOMEM;
    if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
    pCtx->pPhrase = pPhrase = pNew;
    pNew->nTerm = nNew - SZALLOC;
  }

  pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  memset(pTerm, 0, sizeof(Fts5ExprTerm));
  pTerm->zTerm = fts5Strndup(&rc, pToken, nToken);

  return rc;
}


/*
** Free the phrase object passed as the only argument.
*/
void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){
Changes to ext/fts5/fts5_storage.c.
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80











81
82
83
84
85
86
87
88
89
90
91
92
93
94
95


96

97

98
99
100
101
102
103
104
105
106
107
108
109
110
  sqlite3_stmt **ppStmt           /* OUT: Prepared statement handle */
){
  int rc = SQLITE_OK;

  assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
  if( p->aStmt[eStmt]==0 ){
    const char *azStmt[] = {
      "SELECT * FROM %Q.'%q_content' ORDER BY id ASC",  /* SCAN_ASC */
      "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */
      "SELECT * FROM %Q.'%q_content' WHERE rowid=?",    /* LOOKUP  */

      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
      "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
      "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */

      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */

      "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
    };
    Fts5Config *pConfig = p->pConfig;
    char *zSql = 0;












    if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
      int nCol = pConfig->nCol + 1;
      char *zBind;
      int i;

      zBind = sqlite3_malloc(1 + nCol*2);
      if( zBind ){
        for(i=0; i<nCol; i++){
          zBind[i*2] = '?';
          zBind[i*2 + 1] = ',';
        }
        zBind[i*2-1] = '\0';
        zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind);
        sqlite3_free(zBind);
      }


    }else{

      zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName);

    }

    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
    }
  }

  *ppStmt = p->aStmt[eStmt];
  return rc;
}







|
|
|











|


>
>
>
>
>
>
>
>
>
>
>
|
|
|
|

|
|
|
|
|
|
|
|
|
|
>
>
|
>
|
>





|







57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
  sqlite3_stmt **ppStmt           /* OUT: Prepared statement handle */
){
  int rc = SQLITE_OK;

  assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
  if( p->aStmt[eStmt]==0 ){
    const char *azStmt[] = {
      "SELECT * FROM %s ORDER BY id ASC",               /* SCAN_ASC */
      "SELECT * FROM %s ORDER BY id DESC",              /* SCAN_DESC */
      "SELECT * FROM %s WHERE %s=?",                    /* LOOKUP  */

      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
      "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
      "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */

      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */

      "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
    };
    Fts5Config *pC = p->pConfig;
    char *zSql = 0;

    switch( eStmt ){
      case FTS5_STMT_SCAN_ASC:
      case FTS5_STMT_SCAN_DESC:
        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent);
        break;

      case FTS5_STMT_LOOKUP:
        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
        break;

      case FTS5_STMT_INSERT_CONTENT: 
      case FTS5_STMT_REPLACE_CONTENT: {
        int nCol = pC->nCol + 1;
        char *zBind;
        int i;

        zBind = sqlite3_malloc(1 + nCol*2);
        if( zBind ){
          for(i=0; i<nCol; i++){
            zBind[i*2] = '?';
            zBind[i*2 + 1] = ',';
          }
          zBind[i*2-1] = '\0';
          zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
          sqlite3_free(zBind);
        }
        break;
      }

      default:
        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
        break;
    }

    if( zSql==0 ){
      rc = SQLITE_NOMEM;
    }else{
      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
      sqlite3_free(zSql);
    }
  }

  *ppStmt = p->aStmt[eStmt];
  return rc;
}
186
187
188
189
190
191
192
193
194
195
196
197

198
199
200
201
202
203
204


205
206
207
208
209
210
211

  memset(p, 0, nByte);
  p->aTotalSize = (i64*)&p[1];
  p->pConfig = pConfig;
  p->pIndex = pIndex;

  if( bCreate ){
    int i;
    char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
    if( zDefn==0 ){
      rc = SQLITE_NOMEM;
    }else{

      int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
      for(i=0; i<pConfig->nCol; i++){
        iOff += sprintf(&zDefn[iOff], ", c%d", i);
      }
      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
    }
    sqlite3_free(zDefn);


    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
      );
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(







|
|
|
|
|
>
|
|
|
|
|
|
|
>
>







201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229

  memset(p, 0, nByte);
  p->aTotalSize = (i64*)&p[1];
  p->pConfig = pConfig;
  p->pIndex = pIndex;

  if( bCreate ){
    if( pConfig->bExternalContent==0 ){
      char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
      if( zDefn==0 ){
        rc = SQLITE_NOMEM;
      }else{
        int i;
        int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
        for(i=0; i<pConfig->nCol; i++){
          iOff += sprintf(&zDefn[iOff], ", c%d", i);
        }
        rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
      }
      sqlite3_free(zDefn);
    }

    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
      );
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
427
428
429
430
431
432
433








































































434
435
436
437
438
439
440
  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}









































































/*
** Insert a new row into the FTS table.
*/
int sqlite3Fts5StorageInsert(
  Fts5Storage *p,                 /* Storage module to write to */
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;
}

int sqlite3Fts5StorageSpecialDelete(
  Fts5Storage *p, 
  i64 iDel, 
  sqlite3_value **apVal
){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel;

  assert( p->pConfig->bExternalContent );
  rc = fts5StorageLoadTotals(p, 1);

  /* Delete the index records */
  if( rc==SQLITE_OK ){
    int iCol;
    Fts5InsertCtx ctx;
    ctx.pStorage = p;
    ctx.iCol = -1;

    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
      rc = sqlite3Fts5Tokenize(pConfig, 
        (const char*)sqlite3_value_text(apVal[iCol]),
        sqlite3_value_bytes(apVal[iCol]),
        (void*)&ctx,
        fts5StorageInsertCallback
      );
      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
    }
    p->nTotalRow--;
  }

  /* Delete the %_docsize record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
  }
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pDel, 1, iDel);
    sqlite3_step(pDel);
    rc = sqlite3_reset(pDel);
  }

  /* Write the averages record */
  if( rc==SQLITE_OK ){
    rc = fts5StorageSaveTotals(p);
  }

  return rc;

}

/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated
** by inserting a dummy row into the %_docsize table. The dummy will be
** overwritten later.
*/
static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
  if( rc==SQLITE_OK ){
    sqlite3_bind_null(pReplace, 1);
    sqlite3_bind_null(pReplace, 2);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  if( rc==SQLITE_OK ){
    *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
  }
  return rc;
}

/*
** Insert a new row into the FTS table.
*/
int sqlite3Fts5StorageInsert(
  Fts5Storage *p,                 /* Storage module to write to */
  sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
449
450
451
452
453
454
455
456







457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476


477
478
479
480
481
482
483
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  rc = fts5StorageLoadTotals(p, 1);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK ){







    if( eConflict==SQLITE_REPLACE ){
      eStmt = FTS5_STMT_REPLACE_CONTENT;
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
      }
    }else{
      eStmt = FTS5_STMT_INSERT_CONTENT;
    }
  }
  if( rc==SQLITE_OK ){
    rc = fts5StorageGetStmt(p, eStmt, &pInsert);
  }
  for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
    rc = sqlite3_bind_value(pInsert, i, apVal[i]);
  }
  if( rc==SQLITE_OK ){
    sqlite3_step(pInsert);
    rc = sqlite3_reset(pInsert);
  }
  *piRowid = sqlite3_last_insert_rowid(pConfig->db);



  /* Add new entries to the FTS index */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
    ctx.pStorage = p;
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){







|
>
>
>
>
>
>
>
|
|
|
|
|
|
|
|
<
|
|
|
|
|
|
|
|
|
|
|
>
>







539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561

562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
  Fts5InsertCtx ctx;              /* Tokenization callback context object */
  Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */

  memset(&buf, 0, sizeof(Fts5Buffer));
  rc = fts5StorageLoadTotals(p, 1);

  /* Insert the new row into the %_content table. */
  if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){
    if( pConfig->bExternalContent ){
      if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
        *piRowid = sqlite3_value_int64(apVal[1]);
      }else{
        rc = fts5StorageNewRowid(p, piRowid);
      }
    }else{
      if( eConflict==SQLITE_REPLACE ){
        eStmt = FTS5_STMT_REPLACE_CONTENT;
        if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
          rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
        }
      }else{
        eStmt = FTS5_STMT_INSERT_CONTENT;
      }

      if( rc==SQLITE_OK ){
        rc = fts5StorageGetStmt(p, eStmt, &pInsert);
      }
      for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
      }
      if( rc==SQLITE_OK ){
        sqlite3_step(pInsert);
        rc = sqlite3_reset(pInsert);
      }
      *piRowid = sqlite3_last_insert_rowid(pConfig->db);
    }
  }

  /* Add new entries to the FTS index */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
    ctx.pStorage = p;
  }
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){