/ Check-in [17ef5b59]
Login
SQLite training in Houston TX on 2019-11-05 (details)
Part of the 2019 Tcl Conference

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add support for external content tables to fts5.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: 17ef5b59f789e9fa35c4f053246d819987fd06f8
User & Date: dan 2015-01-03 20:44:58
Context
2015-01-05
20:41
Tests and fixes for fts5 external content tables. check-in: 047aaf83 user: dan tags: fts5
2015-01-03
20:44
Add support for external content tables to fts5. check-in: 17ef5b59 user: dan tags: fts5
2015-01-02
14:55
Allow the rank column to be remapped on a per-query basis by including a term similar to "rank match 'bm25(10,2)'" in a where clause. check-in: 1cd15a17 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

  1051   1051         }else{
  1052   1052           rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
  1053   1053         }
  1054   1054       }
  1055   1055     }
  1056   1056     return rc;
  1057   1057   }
         1058  +
         1059  +static int fts5SpecialDelete(
         1060  +  Fts5Table *pTab, 
         1061  +  sqlite3_value **apVal, 
         1062  +  sqlite3_int64 *piRowid
         1063  +){
         1064  +  int rc = SQLITE_OK;
         1065  +  int eType1 = sqlite3_value_type(apVal[1]);
         1066  +  if( eType1==SQLITE_INTEGER ){
         1067  +    sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]);
         1068  +    rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]);
         1069  +  }
         1070  +  return rc;
         1071  +}
  1058   1072   
  1059   1073   /* 
  1060   1074   ** This function is the implementation of the xUpdate callback used by 
  1061   1075   ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
  1062   1076   ** inserted, updated or deleted.
  1063   1077   */
  1064   1078   static int fts5UpdateMethod(
................................................................................
  1082   1096     **   1. The "old" rowid, or NULL.
  1083   1097     **   2. The "new" rowid.
  1084   1098     **   3. Values for each of the nCol matchable columns.
  1085   1099     **   4. Values for the two hidden columns (<tablename> and "rank").
  1086   1100     */
  1087   1101     assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
  1088   1102   
  1089         -  if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
  1090         -    return fts5SpecialInsert(pTab, 
  1091         -        apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
  1092         -    );
         1103  +  if( nArg>1 ){
         1104  +    sqlite3_value *pCmd = sqlite3_value_type(apVal[2 + pConfig->nCol]);
         1105  +    if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
         1106  +      const char *z = sqlite3_value_text(pCmd);
         1107  +      if( pConfig->bExternalContent && sqlite3_stricmp("delete", z) ){
         1108  +        return fts5SpecialDelete(pTab, apVal, pRowid);
         1109  +      }else{
         1110  +        return fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]);
         1111  +      }
         1112  +    }
  1093   1113     }
  1094   1114   
  1095   1115     eType0 = sqlite3_value_type(apVal[0]);
  1096   1116     eConflict = sqlite3_vtab_on_conflict(pConfig->db);
  1097   1117   
  1098   1118     assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  1099         -  if( eType0==SQLITE_INTEGER ){
         1119  +  assert( pVtab->zErrMsg==0 );
         1120  +
         1121  +  if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
  1100   1122       i64 iDel = sqlite3_value_int64(apVal[0]);    /* Rowid to delete */
  1101   1123       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
  1102   1124     }
  1103   1125   
  1104   1126     if( rc==SQLITE_OK && nArg>1 ){
  1105   1127       rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid);
  1106   1128     }

Changes to ext/fts5/fts5Int.h.

    72     72     sqlite3 *db;                    /* Database handle */
    73     73     char *zDb;                      /* Database holding FTS index (e.g. "main") */
    74     74     char *zName;                    /* Name of FTS index */
    75     75     int nCol;                       /* Number of columns */
    76     76     char **azCol;                   /* Column names */
    77     77     int nPrefix;                    /* Number of prefix indexes */
    78     78     int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
           79  +  int bExternalContent;           /* Content is external */
           80  +  char *zContent;                 /* "content=" option value (or NULL) */ 
           81  +  char *zContentRowid;            /* "content_rowid=" option value (or NULL) */ 
    79     82     Fts5Tokenizer *pTok;
    80     83     fts5_tokenizer *pTokApi;
    81     84   
    82     85     /* Values loaded from the %_config table */
    83     86     int iCookie;                    /* Incremented when %_config is modified */
    84     87     int pgsz;                       /* Approximate page size used in %_data */
    85     88     int nAutomerge;                 /* 'automerge' setting */
................................................................................
   405    408   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
   406    409   int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
   407    410   
   408    411   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
   409    412   int sqlite3Fts5StorageRollback(Fts5Storage *p);
   410    413   
   411    414   int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
          415  +
          416  +int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
   412    417   
   413    418   /*
   414    419   ** End of interface to code in fts5_storage.c.
   415    420   **************************************************************************/
   416    421   
   417    422   
   418    423   /**************************************************************************

Changes to ext/fts5/fts5_config.c.

   187    187     int n = strlen(z);
   188    188     while( n>0 && fts5_iswhitespace(z[n-1]) ){
   189    189       z[--n] = '\0';
   190    190     }
   191    191     while( fts5_iswhitespace(*z) ) z++;
   192    192     return z;
   193    193   }
          194  +
          195  +/*
          196  +** Duplicate the string passed as the only argument into a buffer allocated
          197  +** by sqlite3_malloc().
          198  +**
          199  +** Return 0 if an OOM error is encountered.
          200  +*/
          201  +static char *fts5Strdup(int *pRc, const char *z){
          202  +  char *pRet = 0;
          203  +  if( *pRc==SQLITE_OK ){
          204  +    pRet = sqlite3_mprintf("%s", z);
          205  +    if( pRet==0 ) *pRc = SQLITE_NOMEM;
          206  +  }
          207  +  return pRet;
          208  +}
          209  +
          210  +/*
          211  +** Argument z points to a nul-terminated string containing an SQL identifier.
          212  +** This function returns a copy of the identifier enclosed in backtick 
          213  +** quotes.
          214  +*/
          215  +static char *fts5EscapeName(int *pRc, const char *z){
          216  +  char *pRet = 0;
          217  +  if( *pRc==SQLITE_OK ){
          218  +    int n = strlen(z);
          219  +    pRet = (char*)sqlite3_malloc(2 * 2*n + 1);
          220  +    if( pRet==0 ){
          221  +      *pRc = SQLITE_NOMEM;
          222  +    }else{
          223  +      int i;
          224  +      char *p = pRet;
          225  +      for(i=0; i<n; i++){
          226  +        if( z[i]=='`' ) *p++ = '`';
          227  +        *p++ = z[i];
          228  +      }
          229  +      *p++ = '`';
          230  +      *p++ = '\0';
          231  +    }
          232  +  }
          233  +  return pRet;
          234  +}
   194    235   
   195    236   /*
   196    237   ** Parse the "special" CREATE VIRTUAL TABLE directive and update
   197    238   ** configuration object pConfig as appropriate.
   198    239   **
   199    240   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   200    241   ** an error occurs, an SQLite error code is returned and an error message
................................................................................
   286    327         }
   287    328       }
   288    329   
   289    330       sqlite3_free(azArg);
   290    331       sqlite3_free(pDel);
   291    332       return rc;
   292    333     }
          334  +
          335  +  if( sqlite3_strnicmp("content", zCmd, nCmd)==0 ){
          336  +    int rc = SQLITE_OK;
          337  +    if( pConfig->zContent ){
          338  +      *pzErr = sqlite3_mprintf("multiple content=... directives");
          339  +      rc = SQLITE_ERROR;
          340  +    }else{
          341  +      pConfig->zContent = sqlite3_mprintf("%Q.%Q", pConfig->zDb, zArg);
          342  +      pConfig->bExternalContent = 1;
          343  +      if( pConfig->zContent==0 ) rc = SQLITE_NOMEM;
          344  +    }
          345  +    return rc;
          346  +  }
          347  +
          348  +  if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
          349  +    int rc = SQLITE_OK;
          350  +    if( pConfig->zContentRowid ){
          351  +      *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
          352  +      rc = SQLITE_ERROR;
          353  +    }else{
          354  +      pConfig->zContentRowid = fts5EscapeName(&rc, zArg);
          355  +    }
          356  +    return rc;
          357  +  }
   293    358   
   294    359     *pzErr = sqlite3_mprintf("unrecognized directive: \"%s\"", zCmd);
   295    360     return SQLITE_ERROR;
   296    361   }
   297    362   
   298         -/*
   299         -** Duplicate the string passed as the only argument into a buffer allocated
   300         -** by sqlite3_malloc().
   301         -**
   302         -** Return 0 if an OOM error is encountered.
   303         -*/
   304         -static char *fts5Strdup(int *pRc, const char *z){
   305         -  char *pRet = 0;
   306         -  if( *pRc==SQLITE_OK ){
   307         -    pRet = sqlite3_mprintf("%s", z);
   308         -    if( pRet==0 ) *pRc = SQLITE_NOMEM;
   309         -  }
   310         -  return pRet;
   311         -}
   312         -
   313    363   /*
   314    364   ** Allocate an instance of the default tokenizer ("simple") at 
   315    365   ** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
   316    366   ** code if an error occurs.
   317    367   */
   318    368   static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
   319    369     assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
................................................................................
   417    467   
   418    468     /* If a tokenizer= option was successfully parsed, the tokenizer has
   419    469     ** already been allocated. Otherwise, allocate an instance of the default
   420    470     ** tokenizer (simple) now.  */
   421    471     if( rc==SQLITE_OK && pRet->pTok==0 ){
   422    472       rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   423    473     }
          474  +
          475  +  /* If no zContent option was specified, fill in the default values. */
          476  +  if( rc==SQLITE_OK && pRet->zContent==0 ){
          477  +    pRet->zContent = sqlite3_mprintf("%Q.'%q_content'", pRet->zDb, pRet->zName);
          478  +    if( pRet->zContent==0 ){
          479  +      rc = SQLITE_NOMEM;
          480  +    }else{
          481  +      sqlite3_free(pRet->zContentRowid);
          482  +      pRet->zContentRowid = 0;
          483  +    }
          484  +  }
          485  +  if( rc==SQLITE_OK && pRet->zContentRowid==0 ){
          486  +    pRet->zContentRowid = fts5Strdup(&rc, "rowid");
          487  +  }
   424    488   
   425    489     if( rc!=SQLITE_OK ){
   426    490       sqlite3Fts5ConfigFree(pRet);
   427    491       *ppOut = 0;
   428    492     }
   429    493     return rc;
   430    494   }
................................................................................
   443    507       for(i=0; i<pConfig->nCol; i++){
   444    508         sqlite3_free(pConfig->azCol[i]);
   445    509       }
   446    510       sqlite3_free(pConfig->azCol);
   447    511       sqlite3_free(pConfig->aPrefix);
   448    512       sqlite3_free(pConfig->zRank);
   449    513       sqlite3_free(pConfig->zRankArgs);
          514  +    sqlite3_free(pConfig->zContent);
          515  +    sqlite3_free(pConfig->zContentRowid);
   450    516       sqlite3_free(pConfig);
   451    517     }
   452    518   }
   453    519   
   454    520   /*
   455    521   ** Call sqlite3_declare_vtab() based on the contents of the configuration
   456    522   ** object passed as the only argument. Return SQLITE_OK if successful, or

Changes to ext/fts5/fts5_expr.c.

  1037   1037   ** Argument pIn points to a buffer of nIn bytes. This function allocates
  1038   1038   ** and returns a new buffer populated with a copy of (pIn/nIn) with a 
  1039   1039   ** nul-terminator byte appended to it.
  1040   1040   **
  1041   1041   ** It is the responsibility of the caller to eventually free the returned
  1042   1042   ** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. 
  1043   1043   */
  1044         -static char *fts5Strndup(const char *pIn, int nIn){
  1045         -  char *zRet = (char*)sqlite3_malloc(nIn+1);
  1046         -  if( zRet ){
  1047         -    memcpy(zRet, pIn, nIn);
  1048         -    zRet[nIn] = '\0';
         1044  +static char *fts5Strndup(int *pRc, const char *pIn, int nIn){
         1045  +  char *zRet = 0;
         1046  +  if( *pRc==SQLITE_OK ){
         1047  +    zRet = (char*)sqlite3_malloc(nIn+1);
         1048  +    if( zRet ){
         1049  +      memcpy(zRet, pIn, nIn);
         1050  +      zRet[nIn] = '\0';
         1051  +    }else{
         1052  +      *pRc = SQLITE_NOMEM;
         1053  +    }
  1049   1054     }
  1050   1055     return zRet;
  1051   1056   }
  1052   1057   
  1053   1058   static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
  1054         -  *pz = fts5Strndup(pToken->p, pToken->n);
  1055         -  if( *pz==0 ) return SQLITE_NOMEM;
  1056         -  return SQLITE_OK;
         1059  +  int rc = SQLITE_OK;
         1060  +  *pz = fts5Strndup(&rc, pToken->p, pToken->n);
         1061  +  return rc;
  1057   1062   }
  1058   1063   
  1059   1064   /*
  1060   1065   ** Free the phrase object passed as the only argument.
  1061   1066   */
  1062   1067   static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
  1063   1068     if( pPhrase ){
................................................................................
  1135   1140     void *pContext,                 /* Pointer to Fts5InsertCtx object */
  1136   1141     const char *pToken,             /* Buffer containing token */
  1137   1142     int nToken,                     /* Size of token in bytes */
  1138   1143     int iStart,                     /* Start offset of token */
  1139   1144     int iEnd,                       /* End offset of token */
  1140   1145     int iPos                        /* Position offset of token */
  1141   1146   ){
         1147  +  int rc = SQLITE_OK;
  1142   1148     const int SZALLOC = 8;
  1143   1149     TokenCtx *pCtx = (TokenCtx*)pContext;
  1144   1150     Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
  1145   1151     Fts5ExprTerm *pTerm;
  1146   1152   
  1147   1153     if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
  1148   1154       Fts5ExprPhrase *pNew;
................................................................................
  1155   1161       if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
  1156   1162       pCtx->pPhrase = pPhrase = pNew;
  1157   1163       pNew->nTerm = nNew - SZALLOC;
  1158   1164     }
  1159   1165   
  1160   1166     pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
  1161   1167     memset(pTerm, 0, sizeof(Fts5ExprTerm));
  1162         -  pTerm->zTerm = fts5Strndup(pToken, nToken);
         1168  +  pTerm->zTerm = fts5Strndup(&rc, pToken, nToken);
  1163   1169   
  1164         -  return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
         1170  +  return rc;
  1165   1171   }
  1166   1172   
  1167   1173   
  1168   1174   /*
  1169   1175   ** Free the phrase object passed as the only argument.
  1170   1176   */
  1171   1177   void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase *pPhrase){

Changes to ext/fts5/fts5_storage.c.

    57     57     sqlite3_stmt **ppStmt           /* OUT: Prepared statement handle */
    58     58   ){
    59     59     int rc = SQLITE_OK;
    60     60   
    61     61     assert( eStmt>=0 && eStmt<ArraySize(p->aStmt) );
    62     62     if( p->aStmt[eStmt]==0 ){
    63     63       const char *azStmt[] = {
    64         -      "SELECT * FROM %Q.'%q_content' ORDER BY id ASC",  /* SCAN_ASC */
    65         -      "SELECT * FROM %Q.'%q_content' ORDER BY id DESC", /* SCAN_DESC */
    66         -      "SELECT * FROM %Q.'%q_content' WHERE rowid=?",    /* LOOKUP  */
           64  +      "SELECT * FROM %s ORDER BY id ASC",               /* SCAN_ASC */
           65  +      "SELECT * FROM %s ORDER BY id DESC",              /* SCAN_DESC */
           66  +      "SELECT * FROM %s WHERE %s=?",                    /* LOOKUP  */
    67     67   
    68     68         "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
    69     69         "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
    70     70         "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
    71     71         "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
    72     72         "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
    73     73   
    74     74         "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
    75     75   
    76     76         "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
    77     77       };
    78         -    Fts5Config *pConfig = p->pConfig;
           78  +    Fts5Config *pC = p->pConfig;
    79     79       char *zSql = 0;
    80     80   
    81         -    if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
    82         -      int nCol = pConfig->nCol + 1;
    83         -      char *zBind;
    84         -      int i;
           81  +    switch( eStmt ){
           82  +      case FTS5_STMT_SCAN_ASC:
           83  +      case FTS5_STMT_SCAN_DESC:
           84  +        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent);
           85  +        break;
    85     86   
    86         -      zBind = sqlite3_malloc(1 + nCol*2);
    87         -      if( zBind ){
    88         -        for(i=0; i<nCol; i++){
    89         -          zBind[i*2] = '?';
    90         -          zBind[i*2 + 1] = ',';
           87  +      case FTS5_STMT_LOOKUP:
           88  +        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContent, pC->zContentRowid);
           89  +        break;
           90  +
           91  +      case FTS5_STMT_INSERT_CONTENT: 
           92  +      case FTS5_STMT_REPLACE_CONTENT: {
           93  +        int nCol = pC->nCol + 1;
           94  +        char *zBind;
           95  +        int i;
           96  +
           97  +        zBind = sqlite3_malloc(1 + nCol*2);
           98  +        if( zBind ){
           99  +          for(i=0; i<nCol; i++){
          100  +            zBind[i*2] = '?';
          101  +            zBind[i*2 + 1] = ',';
          102  +          }
          103  +          zBind[i*2-1] = '\0';
          104  +          zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
          105  +          sqlite3_free(zBind);
    91    106           }
    92         -        zBind[i*2-1] = '\0';
    93         -        zSql = sqlite3_mprintf(azStmt[eStmt],pConfig->zDb,pConfig->zName,zBind);
    94         -        sqlite3_free(zBind);
          107  +        break;
    95    108         }
    96         -    }else{
    97         -      zSql = sqlite3_mprintf(azStmt[eStmt], pConfig->zDb, pConfig->zName);
          109  +
          110  +      default:
          111  +        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
          112  +        break;
    98    113       }
    99    114   
   100    115       if( zSql==0 ){
   101    116         rc = SQLITE_NOMEM;
   102    117       }else{
   103         -      rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->aStmt[eStmt], 0);
          118  +      rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0);
   104    119         sqlite3_free(zSql);
   105    120       }
   106    121     }
   107    122   
   108    123     *ppStmt = p->aStmt[eStmt];
   109    124     return rc;
   110    125   }
................................................................................
   186    201   
   187    202     memset(p, 0, nByte);
   188    203     p->aTotalSize = (i64*)&p[1];
   189    204     p->pConfig = pConfig;
   190    205     p->pIndex = pIndex;
   191    206   
   192    207     if( bCreate ){
   193         -    int i;
   194         -    char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
   195         -    if( zDefn==0 ){
   196         -      rc = SQLITE_NOMEM;
   197         -    }else{
   198         -      int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
   199         -      for(i=0; i<pConfig->nCol; i++){
   200         -        iOff += sprintf(&zDefn[iOff], ", c%d", i);
          208  +    if( pConfig->bExternalContent==0 ){
          209  +      char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10);
          210  +      if( zDefn==0 ){
          211  +        rc = SQLITE_NOMEM;
          212  +      }else{
          213  +        int i;
          214  +        int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
          215  +        for(i=0; i<pConfig->nCol; i++){
          216  +          iOff += sprintf(&zDefn[iOff], ", c%d", i);
          217  +        }
          218  +        rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
   201    219         }
   202         -      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
          220  +      sqlite3_free(zDefn);
   203    221       }
   204         -    sqlite3_free(zDefn);
          222  +
   205    223       if( rc==SQLITE_OK ){
   206    224         rc = sqlite3Fts5CreateTable(
   207    225             pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
   208    226         );
   209    227       }
   210    228       if( rc==SQLITE_OK ){
   211    229         rc = sqlite3Fts5CreateTable(
................................................................................
   427    445     /* Write the averages record */
   428    446     if( rc==SQLITE_OK ){
   429    447       rc = fts5StorageSaveTotals(p);
   430    448     }
   431    449   
   432    450     return rc;
   433    451   }
          452  +
          453  +int sqlite3Fts5StorageSpecialDelete(
          454  +  Fts5Storage *p, 
          455  +  i64 iDel, 
          456  +  sqlite3_value **apVal
          457  +){
          458  +  Fts5Config *pConfig = p->pConfig;
          459  +  int rc;
          460  +  sqlite3_stmt *pDel;
          461  +
          462  +  assert( p->pConfig->bExternalContent );
          463  +  rc = fts5StorageLoadTotals(p, 1);
          464  +
          465  +  /* Delete the index records */
          466  +  if( rc==SQLITE_OK ){
          467  +    int iCol;
          468  +    Fts5InsertCtx ctx;
          469  +    ctx.pStorage = p;
          470  +    ctx.iCol = -1;
          471  +
          472  +    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel);
          473  +    for(iCol=0; rc==SQLITE_OK && iCol<pConfig->nCol; iCol++){
          474  +      rc = sqlite3Fts5Tokenize(pConfig, 
          475  +        (const char*)sqlite3_value_text(apVal[iCol]),
          476  +        sqlite3_value_bytes(apVal[iCol]),
          477  +        (void*)&ctx,
          478  +        fts5StorageInsertCallback
          479  +      );
          480  +      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
          481  +    }
          482  +    p->nTotalRow--;
          483  +  }
          484  +
          485  +  /* Delete the %_docsize record */
          486  +  if( rc==SQLITE_OK ){
          487  +    rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_DOCSIZE, &pDel);
          488  +  }
          489  +  if( rc==SQLITE_OK ){
          490  +    sqlite3_bind_int64(pDel, 1, iDel);
          491  +    sqlite3_step(pDel);
          492  +    rc = sqlite3_reset(pDel);
          493  +  }
          494  +
          495  +  /* Write the averages record */
          496  +  if( rc==SQLITE_OK ){
          497  +    rc = fts5StorageSaveTotals(p);
          498  +  }
          499  +
          500  +  return rc;
          501  +
          502  +}
          503  +
          504  +/*
          505  +** Allocate a new rowid. This is used for "external content" tables when
          506  +** a NULL value is inserted into the rowid column. The new rowid is allocated
          507  +** by inserting a dummy row into the %_docsize table. The dummy will be
          508  +** overwritten later.
          509  +*/
          510  +static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
          511  +  sqlite3_stmt *pReplace = 0;
          512  +  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace);
          513  +  if( rc==SQLITE_OK ){
          514  +    sqlite3_bind_null(pReplace, 1);
          515  +    sqlite3_bind_null(pReplace, 2);
          516  +    sqlite3_step(pReplace);
          517  +    rc = sqlite3_reset(pReplace);
          518  +  }
          519  +  if( rc==SQLITE_OK ){
          520  +    *piRowid = sqlite3_last_insert_rowid(p->pConfig->db);
          521  +  }
          522  +  return rc;
          523  +}
   434    524   
   435    525   /*
   436    526   ** Insert a new row into the FTS table.
   437    527   */
   438    528   int sqlite3Fts5StorageInsert(
   439    529     Fts5Storage *p,                 /* Storage module to write to */
   440    530     sqlite3_value **apVal,          /* Array of values passed to xUpdate() */
................................................................................
   449    539     Fts5InsertCtx ctx;              /* Tokenization callback context object */
   450    540     Fts5Buffer buf;                 /* Buffer used to build up %_docsize blob */
   451    541   
   452    542     memset(&buf, 0, sizeof(Fts5Buffer));
   453    543     rc = fts5StorageLoadTotals(p, 1);
   454    544   
   455    545     /* Insert the new row into the %_content table. */
   456         -  if( rc==SQLITE_OK ){
   457         -    if( eConflict==SQLITE_REPLACE ){
   458         -      eStmt = FTS5_STMT_REPLACE_CONTENT;
          546  +  if( rc==SQLITE_OK && pConfig->bExternalContent==0 ){
          547  +    if( pConfig->bExternalContent ){
   459    548         if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
   460         -        rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
          549  +        *piRowid = sqlite3_value_int64(apVal[1]);
          550  +      }else{
          551  +        rc = fts5StorageNewRowid(p, piRowid);
   461    552         }
   462    553       }else{
   463         -      eStmt = FTS5_STMT_INSERT_CONTENT;
          554  +      if( eConflict==SQLITE_REPLACE ){
          555  +        eStmt = FTS5_STMT_REPLACE_CONTENT;
          556  +        if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
          557  +          rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1]));
          558  +        }
          559  +      }else{
          560  +        eStmt = FTS5_STMT_INSERT_CONTENT;
          561  +      }
          562  +      if( rc==SQLITE_OK ){
          563  +        rc = fts5StorageGetStmt(p, eStmt, &pInsert);
          564  +      }
          565  +      for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
          566  +        rc = sqlite3_bind_value(pInsert, i, apVal[i]);
          567  +      }
          568  +      if( rc==SQLITE_OK ){
          569  +        sqlite3_step(pInsert);
          570  +        rc = sqlite3_reset(pInsert);
          571  +      }
          572  +      *piRowid = sqlite3_last_insert_rowid(pConfig->db);
   464    573       }
   465    574     }
   466         -  if( rc==SQLITE_OK ){
   467         -    rc = fts5StorageGetStmt(p, eStmt, &pInsert);
   468         -  }
   469         -  for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
   470         -    rc = sqlite3_bind_value(pInsert, i, apVal[i]);
   471         -  }
   472         -  if( rc==SQLITE_OK ){
   473         -    sqlite3_step(pInsert);
   474         -    rc = sqlite3_reset(pInsert);
   475         -  }
   476         -  *piRowid = sqlite3_last_insert_rowid(pConfig->db);
   477    575   
   478    576     /* Add new entries to the FTS index */
   479    577     if( rc==SQLITE_OK ){
   480    578       rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid);
   481    579       ctx.pStorage = p;
   482    580     }
   483    581     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){