SQLite

    );
  }

  /* Call sqlite3_declare_vtab() */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigDeclareVtab(pConfig);
  }

  /* Load the contents of %_config */
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5ConfigLoad(pConfig);
  }

  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab, 0);
    pTab = 0;
  }else if( bCreate ){
    fts5CheckTransactionState(pTab, FTS5_BEGIN, 0);
  }

  return rc;
}

/*
** This function is called to handle an FTS INSERT command. In other words,
** an INSERT statement of the form:
**
**     INSERT INTO fts(fts) VALUES($pCmd)
**     INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal)
**
** Argument pVal is the value assigned to column "fts" by the INSERT 
** statement. This function returns SQLITE_OK if successful, or an SQLite
** error code if an error occurs.
**
** The commands implemented by this function are documented in the "Special
** INSERT Directives" section of the documentation. It should be updated if
** more commands are added to this function.
*/
static int fts5SpecialCommand(
  Fts5Table *pTab,                /* Fts5 table object */
  sqlite3_value *pCmd,            /* Value inserted into special column */
  sqlite3_value *pVal             /* Value inserted into rowid column */
){
  const char *z = (const char*)sqlite3_value_text(pCmd);

  int rc = SQLITE_OK;
  int bError = 0;

  if( 0==sqlite3_stricmp("integrity-check", z) ){
    rc = sqlite3Fts5StorageIntegrity(pTab->pStorage);
  }else{
    rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);


    if( rc==SQLITE_OK && bError ){

      rc = SQLITE_ERROR;
    }else{
      rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);




    }
  }
  return rc;
}

/* 
** This function is the implementation of the xUpdate callback used by 
** FTS3 virtual tables. It is invoked by SQLite each time a row is to be
** inserted, updated or deleted.

  **   2. The "new" rowid.
  **   3. Values for each of the nCol matchable columns.
  **   4. Values for the two hidden columns (<tablename> and "rank").
  */
  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );

  if( nArg>1 && SQLITE_NULL!=sqlite3_value_type(apVal[2 + pConfig->nCol]) ){
    return fts5SpecialCommand(pTab, 
        apVal[2 + pConfig->nCol], apVal[2 + pConfig->nCol + 1]
    );
  }

  eType0 = sqlite3_value_type(apVal[0]);
  eConflict = sqlite3_vtab_on_conflict(pConfig->db);

  assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
  if( eType0==SQLITE_INTEGER ){

        sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[i]);
      }

      while( 1 ){
        int *aInst;
        int iBest = -1;
        for(i=0; i<nIter; i++){
          if( (aIter[i].bEof==0) 
           && (iBest<0 || aIter[i].iPos<aIter[iBest].iPos) 
          ){
            iBest = i;
          }
        }

        if( iBest<0 ) break;
        nInst++;
        if( sqlite3Fts5BufferGrow(&rc, &buf, nInst * sizeof(int) * 3) ) break;

*/

typedef struct Fts5Config Fts5Config;

/*
** An instance of the following structure encodes all information that can
** be gleaned from the CREATE VIRTUAL TABLE statement.
**
** And all information loaded from the %_config table.
*/
struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
  char **azCol;                   /* Column names */
  int nPrefix;                    /* Number of prefix indexes */
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  Fts5Tokenizer *pTok;
  fts5_tokenizer *pTokApi;

  /* Values loaded from the %_config table */
  int iCookie;                    /* Incremented when %_config is modified */
  int pgsz;                       /* Approximate page size used in %_data */
};

int sqlite3Fts5ConfigParse(
    Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
);
void sqlite3Fts5ConfigFree(Fts5Config*);

int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);

int sqlite3Fts5Tokenize(
  Fts5Config *pConfig,            /* FTS5 Configuration object */
  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
);

void sqlite3Fts5Dequote(char *z);

/* Load the contents of the %_config table */
int sqlite3Fts5ConfigLoad(Fts5Config*);

/* Set the value of a single config attribute */
int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);

/*
** End of interface to code in fts5_config.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_buffer.c.
*/

/* 
** Called during virtual module initialization to register UDF 
** fts5_decode() with SQLite 
*/
int sqlite3Fts5IndexInit(sqlite3*);








void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMerge);

/*
** Return the total number of entries read from the %_data table by 
** this connection since it was created.
*/
int sqlite3Fts5IndexReads(Fts5Index *p);

typedef struct Fts5Storage Fts5Storage;

int sqlite3Fts5StorageOpen(Fts5Config*, Fts5Index*, int, Fts5Storage**, char**);
int sqlite3Fts5StorageClose(Fts5Storage *p, int bDestroy);

int sqlite3Fts5DropTable(Fts5Config*, const char *zPost);
int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

int sqlite3Fts5StorageDelete(Fts5Storage *p, i64);
int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*);

int sqlite3Fts5StorageIntegrity(Fts5Storage *p);

int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt **);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);

int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
int sqlite3Fts5StorageRollback(Fts5Storage *p);

int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);

/*
** End of interface to code in fts5_storage.c.
**************************************************************************/


/**************************************************************************

/*************************************************************************
** Start of highlight() implementation.
*/
typedef struct HighlightContext HighlightContext;
struct HighlightContext {
  const Fts5ExtensionApi *pApi;   /* API offered by current FTS version */
  Fts5Context *pFts;              /* First arg to pass to pApi functions */
  int nInst;                      /* Total number of phrase instances */
  int iInst;                      /* Current phrase instance index */
  int iStart;                     /* First token of current phrase */
  int iEnd;                       /* Last token of current phrase */

  const char *zOpen;              /* Opening highlight */
  const char *zClose;             /* Closing highlight */
  int iCol;                       /* Column to read from */

  const char *zIn;                /* Input text */
  int nIn;                        /* Size of input text in bytes */
  int iOff;                       /* Current offset within zIn[] */
  char *zOut;                     /* Output value */
};

/*
** Append text to the HighlightContext output string - p->zOut. Argument
** z points to a buffer containing n bytes of text to append. If n is 
** negative, everything up until the first '\0' is appended to the output.
*/
static void fts5HighlightAppend(
  int *pRc, 
  HighlightContext *p, 
  const char *z, int n
){
  if( *pRc==SQLITE_OK ){
    if( n<0 ) n = strlen(z);
    p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z);
    if( p->zOut==0 ) *pRc = SQLITE_NOMEM;

  }
}

static int fts5HighlightCb(
  void *pContext,                 /* Pointer to HighlightContext object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStartOff,                  /* Start offset of token */
  int iEndOff,                    /* End offset of token */
  int iPos                        /* Position offset of token */
){
  HighlightContext *p = (HighlightContext*)pContext;
  int rc = SQLITE_OK;

  if( iPos==p->iStart ){
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);


    fts5HighlightAppend(&rc, p, p->zOpen, -1);





    p->iOff = iStartOff;
  }

  if( iPos==p->iEnd ){
    int bClose = 1;

    for(p->iInst++; rc==SQLITE_OK && p->iInst<p->nInst; p->iInst++){
      int iP, iPCol, iOff;
      rc = p->pApi->xInst(p->pFts, p->iInst, &iP, &iPCol, &iOff);
      if( iPCol!=p->iCol ){
        p->iStart = p->iEnd = -1;


      }else{
        int iEnd = iOff - 1 + p->pApi->xPhraseSize(p->pFts, iP);
        if( iEnd<=p->iEnd ) continue;
        if( iOff<=p->iEnd ) bClose = 0;
        p->iStart = iOff;
        p->iEnd = iEnd;
      }
      break;
    }

    if( bClose ){
      fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
      fts5HighlightAppend(&rc, p, p->zClose, -1);
      p->iOff = iEndOff;
    }
  }

  return rc;
}

static void fts5HighlightFunction(

  int rc;

  if( nVal!=3 ){
    const char *zErr = "wrong number of arguments to function highlight()";
    sqlite3_result_error(pCtx, zErr, -1);
    return;
  }

  memset(&ctx, 0, sizeof(HighlightContext));
  ctx.iCol = sqlite3_value_int(apVal[0]);
  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);

  ctx.pApi = pApi;
  ctx.pFts = pFts;
  rc = pApi->xColumnText(pFts, ctx.iCol, &ctx.zIn, &ctx.nIn);
  if( rc==SQLITE_OK ) rc = pApi->xInstCount(pFts, &ctx.nInst);

  /* Find the first phrase instance in the right column. */
  ctx.iStart = -1;
  ctx.iEnd = -1;
  for( ; ctx.iInst<ctx.nInst && rc==SQLITE_OK; ctx.iInst++){
    int iP, iPCol, iOff;
    rc = pApi->xInst(pFts, ctx.iInst, &iP, &iPCol, &iOff);
    if( iPCol==ctx.iCol ){
      ctx.iStart = iOff;
      ctx.iEnd = iOff - 1 + pApi->xPhraseSize(pFts, iP);
      break;
    }

  }

  if( rc==SQLITE_OK ){
    rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx, fts5HighlightCb);
  }

  fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);


  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3_free(ctx.zOut);

******************************************************************************
**
** This is an SQLite module implementing full-text search.
*/

#include "fts5Int.h"

#define FTS5_DEFAULT_PAGE_SIZE   1000

/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters.  The conversion is done in-place.  If the
** input does not begin with a quote character, then this routine
** is a no-op.
**
** Examples:

  const char *pText, int nText,   /* Text to tokenize */
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, const char*, int, int, int, int)    /* Callback */
){
  return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
}

int sqlite3Fts5ConfigSetValue(
  Fts5Config *pConfig, 
  const char *zKey, 
  sqlite3_value *pVal,
  int *pbBadkey
){
  int rc = SQLITE_OK;
  if(      0==sqlite3_stricmp(zKey, "cookie") ){
    pConfig->iCookie = sqlite3_value_int(pVal);
  }
  else if( 0==sqlite3_stricmp(zKey, "pgsz") ){
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      pConfig->pgsz = sqlite3_value_int(pVal);
    }else{
      if( pbBadkey ) *pbBadkey = 1;
    }
  }
  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
    // todo
  }
  else if( 0==sqlite3_stricmp(zKey, "rank") ){
    // todo
  }else{
    if( pbBadkey ) *pbBadkey = 1;
  }
  return rc;
}

/*
** Load the contents of the %_config table into memory.
*/
int sqlite3Fts5ConfigLoad(Fts5Config *pConfig){
  const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
  char *zSql;
  sqlite3_stmt *p = 0;
  int rc;

  /* Set default values */
  pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
  pConfig->iCookie = 0;

  zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p, 0);
    sqlite3_free(zSql);
  }

  assert( rc==SQLITE_OK || p==0 );
  if( rc==SQLITE_OK ){
    while( SQLITE_ROW==sqlite3_step(p) ){
      const char *zK = (const char*)sqlite3_column_text(p, 0);
      sqlite3_value *pVal = sqlite3_column_value(p, 1);
      sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
    }
    rc = sqlite3_finalize(p);
  }

  return rc;
}

**     large doclists with very small doclists.
**
**   * extra fields in the "structure record" record the state of ongoing
**     incremental merge operations.
**
*/



#define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
#define FTS5_MIN_MERGE       4    /* Minimum number of segments to merge */
#define FTS5_CRISIS_MERGE   16    /* Maximum number of segments to merge */

#define FTS5_MIN_DLIDX_SIZE  4    /* Add dlidx if this many empty pages */

/*

/*
** One object per %_data table.
*/
struct Fts5Index {
  Fts5Config *pConfig;            /* Virtual table configuration */
  char *zDataTbl;                 /* Name of %_data table */

  int nMinMerge;                  /* Minimum input segments in a merge */
  int nCrisisMerge;               /* Maximum allowed segments per level */
  int nWorkUnit;                  /* Leaf pages in a "unit" of work */

  /*
  ** Variables related to the accumulation of tokens and doclists within the
  ** in-memory hash tables before they are flushed to disk.

  return 0;
}

/*
** Discard all data currently cached in the hash-tables.
*/
static void fts5IndexDiscardData(Fts5Index *p){
  assert( p->apHash || p->nPendingData==0 );
  if( p->apHash ){
    Fts5Config *pConfig = p->pConfig;
    int i;
    for(i=0; i<=pConfig->nPrefix; i++){
      sqlite3Fts5HashClear(p->apHash[i]);
    }
    p->nPendingData = 0;
  }
}

/*
** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
** with buffer (nOld/pOld).
*/
static int fts5PrefixCompress(

      fts5WriteBtreeGrow(p, pWriter);
      if( p->rc ) return;
    }
    pPage = &pWriter->aWriter[iHeight];

    fts5WriteBtreeNEmpty(p, pWriter);

    if( pPage->buf.n>=p->pConfig->pgsz ){
      /* pPage will be written to disk. The term will be written into the
      ** parent of pPage.  */
      i64 iRowid = FTS5_SEGMENT_ROWID(
          pWriter->iIdx, pWriter->iSegid, iHeight, pPage->pgno
      );
      fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
      fts5BufferZero(&pPage->buf);

  /* Update the Fts5PageWriter.term field. */
  fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm);

  pWriter->bFirstRowidInPage = 0;
  pWriter->bFirstRowidInDoclist = 1;

  /* If the current leaf page is full, flush it to disk. */
  if( pPage->buf.n>=p->pConfig->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}

/*
** Append a docid to the writers output. 

    assert( iRowid<pWriter->iPrevRowid );
    fts5BufferAppendVarint(&p->rc, &pPage->buf, pWriter->iPrevRowid - iRowid);
  }
  pWriter->iPrevRowid = iRowid;
  pWriter->bFirstRowidInDoclist = 0;
  pWriter->bFirstRowidInPage = 0;

  if( pPage->buf.n>=p->pConfig->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}

static void fts5WriteAppendPoslistInt(
  Fts5Index *p, 
  Fts5SegWriter *pWriter,
  int iVal
){
  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  fts5BufferAppendVarint(&p->rc, &pPage->buf, iVal);
  if( pPage->buf.n>=p->pConfig->pgsz ){
    fts5WriteFlushLeaf(p, pWriter);
    pWriter->bFirstRowidInPage = 1;
  }
}

static void fts5WriteAppendPoslistData(
  Fts5Index *p, 
  Fts5SegWriter *pWriter, 
  const u8 *aData, 
  int nData
){
  Fts5PageWriter *pPage = &pWriter->aWriter[0];
  const u8 *a = aData;
  int n = nData;
  
  while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){
    int nReq = p->pConfig->pgsz - pPage->buf.n;
    int nCopy = 0;
    while( nCopy<nReq ){
      i64 dummy;
      nCopy += getVarint(&a[nCopy], (u64*)&dummy);
    }
    fts5BufferAppendBlob(&p->rc, &pPage->buf, nCopy, a);
    a += nCopy;

  Fts5Index *p;                   /* New object */

  *pp = p = (Fts5Index*)sqlite3_malloc(sizeof(Fts5Index));
  if( !p ) return SQLITE_NOMEM;

  memset(p, 0, sizeof(Fts5Index));
  p->pConfig = pConfig;

  p->nMinMerge = FTS5_MIN_MERGE;
  p->nCrisisMerge = FTS5_CRISIS_MERGE;
  p->nWorkUnit = FTS5_WORK_UNIT;
  p->nMaxPendingData = 1024*1024;
  p->zDataTbl = sqlite3_mprintf("%s_data", pConfig->zName);
  if( p->zDataTbl==0 ){
    rc = SQLITE_NOMEM;
  }else if( bCreate ){
    int i;
    Fts5Structure s;
    rc = sqlite3Fts5CreateTable(
        pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr
    );
    if( rc==SQLITE_OK ){
      memset(&s, 0, sizeof(Fts5Structure));
      for(i=0; i<pConfig->nPrefix+1; i++){
        fts5StructureWrite(p, i, &s);
      }
      rc = p->rc;

int sqlite3Fts5IndexInit(sqlite3 *db){
  int rc = sqlite3_create_function(
      db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0
  );
  return rc;
}








/*
** Set the minimum number of segments that an auto-merge operation should
** attempt to merge together. A value of 1 sets the object to use the 
** compile time default. Zero or less disables auto-merge altogether.
*/
void sqlite3Fts5IndexAutomerge(Fts5Index *p, int nMinMerge){
  if( nMinMerge==1 ){

struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
  sqlite3_stmt *aStmt[10];
};


#if FTS5_STMT_SCAN_ASC!=0 
# error "FTS5_STMT_SCAN_ASC mismatch" 
#endif
#if FTS5_STMT_SCAN_DESC!=1 

#define FTS5_STMT_REPLACE_CONTENT 4

#define FTS5_STMT_DELETE_CONTENT  5
#define FTS5_STMT_REPLACE_DOCSIZE  6
#define FTS5_STMT_DELETE_DOCSIZE  7

#define FTS5_STMT_LOOKUP_DOCSIZE  8

#define FTS5_STMT_REPLACE_CONFIG 9

/*
** Prepare the two insert statements - Fts5Storage.pInsertContent and
** Fts5Storage.pInsertDocsize - if they have not already been prepared.
** Return SQLITE_OK if successful, or an SQLite error code if an error
** occurs.
*/

      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
      "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
      "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */

      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */

      "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
    };
    Fts5Config *pConfig = p->pConfig;
    char *zSql = 0;

    if( eStmt==FTS5_STMT_INSERT_CONTENT || eStmt==FTS5_STMT_REPLACE_CONTENT ){
      int nCol = pConfig->nCol + 1;
      char *zBind;

** Create the shadow table named zPost, with definition zDefn. Return
** SQLITE_OK if successful, or an SQLite error code otherwise.
*/
int sqlite3Fts5CreateTable(
  Fts5Config *pConfig,            /* FTS5 configuration */
  const char *zPost,              /* Shadow table to create (e.g. "content") */
  const char *zDefn,              /* Columns etc. for shadow table */
  int bWithout,                   /* True for without rowid */
  char **pzErr                    /* OUT: Error message */
){
  int rc;
  char *zSql = sqlite3_mprintf("CREATE TABLE %Q.'%q_%q'(%s)%s",
      pConfig->zDb, pConfig->zName, zPost, zDefn, 
      (bWithout ? " WITHOUT ROWID" :"")
  );
  if( zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    char *zErr = 0;
    assert( *pzErr==0 );
    rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);

    if( zDefn==0 ){
      rc = SQLITE_NOMEM;
    }else{
      int iOff = sprintf(zDefn, "id INTEGER PRIMARY KEY");
      for(i=0; i<pConfig->nCol; i++){
        iOff += sprintf(&zDefn[iOff], ", c%d", i);
      }
      rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
    }
    sqlite3_free(zDefn);
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
      );
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts5CreateTable(
          pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
      );
    }
  }

  if( rc ){
    sqlite3Fts5StorageClose(p, 0);
    *pp = 0;

  for(i=0; i<ArraySize(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }

  /* If required, remove the shadow tables from the database */
  if( bDestroy ){
    rc = sqlite3Fts5DropTable(p->pConfig, "content");
    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "docsize");
    if( rc==SQLITE_OK ) rc = sqlite3Fts5DropTable(p->pConfig, "config");
  }

  sqlite3_free(p);
  return rc;
}

typedef struct Fts5InsertCtx Fts5InsertCtx;

  return sqlite3Fts5IndexSync(p->pIndex, bCommit);
}

int sqlite3Fts5StorageRollback(Fts5Storage *p){
  p->bTotalsValid = 0;
  return sqlite3Fts5IndexRollback(p->pIndex);
}

int sqlite3Fts5StorageConfigValue(
  Fts5Storage *p, 
  const char *z, 
  sqlite3_value *pVal
){
  sqlite3_stmt *pReplace = 0;
  int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace);
  if( rc==SQLITE_OK ){
    sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_TRANSIENT);
    sqlite3_bind_value(pReplace, 2, pVal);
    sqlite3_step(pReplace);
    rc = sqlite3_reset(pReplace);
  }
  return rc;
}

  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
  SELECT name, sql FROM sqlite_master;
} {
  t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)}
  t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)}
  t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)}
  t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)}
  t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID}
}

do_execsql_test 1.1 {
  DROP TABLE t1;
  SELECT name, sql FROM sqlite_master;
} {
}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}
foreach {i x y} {
   1  {g f d b f} {h h e i a}
   2  {f i g j e} {i j c f f}
   3  {e e i f a} {e h f d f}
   4  {h j f j i} {h a c f j}
   5  {d b j c g} {f e i b e}

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}
foreach {i x y} {
   1  {dd abc abc abc abcde} {aaa dd ddd ddd aab}
   2  {dd aab d aaa b} {abcde c aaa aaa aaa}
   3  {abcde dd b b dd} {abc abc d abc ddddd}
   4  {aaa abcde dddd dddd abcde} {abc b b abcde abc}
   5  {aab dddd d dddd c} {ddd abcde dddd abcde c}

#-------------------------------------------------------------------------
#
breakpoint
reset_db
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

do_execsql_test 6.1 {
  INSERT  INTO t1(rowid, x, y) VALUES(22, 'a b c', 'c b a');
  REPLACE INTO t1(rowid, x, y) VALUES(22, 'd e f', 'f e d');
}

do_execsql_test 6.2 {
  INSERT INTO t1(t1) VALUES('integrity-check') 
}

#-------------------------------------------------------------------------
#
reset_db
expr srand(0)
do_execsql_test 7.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

proc doc {} {
  set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
  set ret [list]
  for {set j 0} {$j < 20} {incr j} {
    lappend ret [lindex $v [expr int(rand()*[llength $v])]]

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

do_execsql_test 8.1 {
  INSERT INTO t1 VALUES('the quick brown fox');
  INSERT INTO t1(t1) VALUES('integrity-check');
}


#-------------------------------------------------------------------------
#
reset_db

expr srand(0)

do_execsql_test 9.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x,y,z, prefix="1,2,3");
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

proc doc {} {
  set v [list aaa aab abc abcde b c d dd ddd dddd ddddd]
  set ret [list]
  for {set j 0} {$j < 20} {incr j} {
    lappend ret [lindex $v [expr int(rand()*[llength $v])]]

} {}

#-------------------------------------------------------------------------

reset_db
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
  INSERT INTO t1 VALUES('one');
  INSERT INTO t1 VALUES('two');
  INSERT INTO t1 VALUES('three');
}

do_catchsql_test 2.2 {
  SELECT rowid, * FROM t1 WHERE t1 MATCH 'AND AND'

}

#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a,b);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

foreach {tn a b} {
   1 {abashed abandons abase abash abaft} {abases abased}
   2 {abasing abases abaft abated abandons} {abases abandoned}
   3 {abatement abash abash abated abase} {abasements abashing}
   4 {abaft abasements abase abasement abasing} {abasement abases}

ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE xx USING fts5(x,y);
  INSERT INTO xx(xx, rowid) VALUES('pgsz', 32);
}

set data {
    0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
    1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
    2   {l s z j k i m p s}                 {l w e j t j e e i t w r o p o}
    3   {x g y m y m h p}                   {k j j b r e y y a k y}

    SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC
  } $res
}

foreach {T create} {
  2 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
  }
  
  3 {
    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
    INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
  }

} {

  do_test $T.1 { 
    execsql { DROP TABLE IF EXISTS t1 }
    execsql $create

ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 32);
}

do_execsql_test 1.1 {
  INSERT INTO t1 VALUES('hello', 'world');
  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1}

#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#

do_test 1.0 {
  execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
  execsql { INSERT INTO t1(t1, rowid) VALUES('pgsz', 128) }
  for {set i 1} {$i <= 10000} {incr i} {
    set v {x x x x x x x x x x x x x x x x x x x x}
    if {($i % 2139)==0} {lset v 3 Y ; lappend Y $i}
    if {($i % 1577)==0} {lset v 5 W ; lappend W $i}
    execsql { INSERT INTO t1 VALUES($v) }
  }
} {}

  }
  set res
}

expr srand(0)
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(x);
  INSERT INTO t1(t1, rowid) VALUES('pgsz', 64);
}

for {set iTest 0} {$iTest < 50000} {incr iTest} {
  if {$iTest > 1000} { execsql { DELETE FROM t1 WHERE rowid=($iTest-1000) } }
  set new [doc]
  execsql { INSERT INTO t1 VALUES($new) }
  if {$iTest==10000} { set sz1 [db one {SELECT count(*) FROM t1_data}] }

}

do_execsql_test 2.5 {
  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
} {
  {a [b c d e] f g h i j}
}

do_execsql_test 2.6.1 {
  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
} {
  {a b c [d] e [f] g h i j}
}

do_execsql_test 2.6.2 {
  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
} {
  {a b c [d] e [f] g h i j}
}


finish_test

# 2014 November 24
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the FTS5 module.
#
# Specifically, this function tests the %_config table.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts5al

# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
  finish_test
  return
}

do_execsql_test 1.1 {
  CREATE VIRTUAL TABLE ft1 USING fts5(x);
  SELECT * FROM ft1_config;
} {}

do_execsql_test 1.2 {
  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
  SELECT * FROM ft1_config;
} {pgsz 32}

do_execsql_test 1.3 {
  INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
  SELECT * FROM ft1_config;
} {pgsz 64}

finish_test