SQLite4  Check-in [e21b7b67b5]

  }
  sqlite4OpenIndex(pParse, iIdx, iDb, pIdx, OP_OpenWrite);
  if( bCreate ) sqlite4VdbeChangeP5(v, 1);

  /* Loop through the contents of the PK index. At each row, insert the
  ** corresponding entry into the auxiliary index.  */
  addr1 = sqlite4VdbeAddOp2(v, OP_Rewind, iTab, 0);

  if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){
    int regData;
    int i;

    regKey = sqlite4GetTempRange(pParse, pTab->nCol+1);
    regData = regKey+1;

    sqlite4VdbeAddOp2(v, OP_RowKey, iTab, regKey);
    for(i=0; i<pTab->nCol; i++){
      sqlite4VdbeAddOp3(v, OP_Column, iTab, i, regData+i);
    }
    sqlite4Fts5CodeUpdate(pParse, pIdx, pParse->iNewidxReg, regKey, regData, 0);
  }else{
    sqlite4GetTempRange(pParse,2);
    regKey = sqlite4GetTempReg(pParse);
    sqlite4EncodeIndexKey(pParse, pPk, iTab, pIdx, iIdx, 0, regKey);
    if( pIdx->onError!=OE_None ){
      const char *zErr = "indexed columns are not unique";
      int addrTest;

      addrTest = sqlite4VdbeAddOp4Int(v, OP_IsUnique, iIdx, 0, regKey, 0);
      sqlite4HaltConstraint(pParse, OE_Abort, (char *)zErr, P4_STATIC);
      sqlite4VdbeJumpHere(v, addrTest);
    }
    sqlite4VdbeAddOp3(v, OP_IdxInsert, iIdx, 0, regKey);  
  }

  sqlite4VdbeAddOp2(v, OP_Next, iTab, addr1+1);
  sqlite4VdbeJumpHere(v, addr1);
  sqlite4ReleaseTempReg(pParse, regKey);

  sqlite4VdbeAddOp1(v, OP_Close, iTab);
  sqlite4VdbeAddOp1(v, OP_Close, iIdx);
}

        addIndexToHash(db, pIdx);
        pIdx = 0;
      }else{
        createIndexWriteSchema(pParse, pIdx, pIdxName, pEnd);
      }
    }

    if( pIdx ) freeIndex(db, pIdx);
    sqlite4DbFree(db, zIdx);
  }

  sqlite4ExprListDelete(db, pList);
  sqlite4SrcListDelete(db, p->pTblName);
}

      int iCol;
      int iReg = pParse->nMem+1;
      pParse->nMem += (1 + pTab->nCol);
      for(iCol=0; iCol<pTab->nCol; iCol++){
        sqlite4VdbeAddOp3(v, OP_Column, iPkCsr, iCol, iReg+iCol);
      }
      sqlite4VdbeAddOp2(v, OP_RowKey, iPkCsr, iReg+pTab->nCol);
      sqlite4Fts5CodeUpdate(pParse, pIdx, 0, iReg+pTab->nCol, iReg, 1);
    }else if( pIdx!=pPk && (aRegIdx==0 || aRegIdx[i]>0) ){
      int addrNotFound;
      sqlite4EncodeIndexKey(pParse, pPk, baseCur+iPk,pIdx,baseCur+i,0,regKey);
      addrNotFound = sqlite4VdbeAddOp4(v,
          OP_NotFound, baseCur+i, 0, regKey, 0, P4_INT32
      );
      sqlite4VdbeAddOp1(v, OP_Delete, baseCur+i);

  KVCursor *pCsr;                 /* Cursor used to retrive values */
  Mem *aMem;                      /* Array of column values */
  int bMemValid;                  /* True if contents of aMem[] are valid */

  Fts5Size *pSz;                  /* Local size data */
  Fts5Size *pGlobal;              /* Global size data */
  i64 nGlobal;                    /* Total number of rows in table */

  /* Arrays used by sqlite4_mi_row_count(). */
  int *anRowCS;
  int *anRowC;
  int *anRowS;
  int *anRow;

  Fts5MatchIter *pIter;           /* Used by mi_match_detail() */
};

/*
** A deserialized 'size record' (see above).

  pToken = &p->pStr->aToken[p->pStr->nToken];

  zSpace = &pParse->aSpace[pParse->iSpace];
  nUsed = putVarint32((u8 *)zSpace, pParse->iRoot);
  zSpace[nUsed++] = 0x24;
  pToken->bPrefix = 0;
  pToken->pPrefix = 0;
  pToken->z = &zSpace[nUsed];
  pToken->n = n;
  memcpy(pToken->z, z, n);
  pToken->z[n] = '\0';

  nUsed += (n+1);
  pToken->aPrefix = (u8 *)zSpace;

  int nToken, 
  int iSrc, 
  int nSrc
){
  TokenizeCtx *p = (TokenizeCtx *)pCtx;
  sqlite4 *db = p->db;
  TokenizeTerm *pTerm = 0;


  /* TODO: Error here if iStream is out of range */

  if( nToken>p->nMax ) p->nMax = nToken;

  if( iStream>=p->nStream ){
    int nOld = p->nStream;

  pTerm = (TokenizeTerm *)sqlite4HashFind(&p->hash, zToken, nToken);
  if( pTerm==0 ){
    /* Size the initial allocation so that it fits in the lookaside buffer */
    int nAlloc = sizeof(TokenizeTerm) + nToken + 32;

    pTerm = sqlite4DbMallocZero(p->db, nAlloc);
    if( pTerm ){

      pTerm->nAlloc = sqlite4DbMallocSize(p->db, pTerm);
      pTerm->nToken = nToken;
      memcpy(&pTerm[1], zToken, nToken);




    }
    if( pTerm==0 ) goto tokenize_cb_out;
  }else{
    sqlite4HashInsert(&p->hash, zToken, nToken, 0);
  }


  if( iStream!=pTerm->iStream ){
    pTerm = fts5TokenizeAppendInt(p, pTerm, (iStream << 2) | 0x00000003);
    if( !pTerm ) goto tokenize_cb_out;
    pTerm->iStream = iStream;
  }

  if( pTerm && p->iCol!=pTerm->iCol ){
    pTerm = fts5TokenizeAppendInt(p, pTerm, (p->iCol << 2) | 0x00000001);
    if( !pTerm ) goto tokenize_cb_out;
    pTerm->iCol = p->iCol;
    pTerm->iOff = 0;
  }

  pTerm = fts5TokenizeAppendInt(p, pTerm, (iOff-pTerm->iOff) << 1);
  if( !pTerm ) goto tokenize_cb_out;
  pTerm->iOff = iOff;

tokenize_cb_out:

  sqlite4HashInsert(&p->hash, (char *)&pTerm[1], nToken, pTerm);

  if( !pTerm ){
    p->rc = SQLITE4_NOMEM;
    return 1;
  }

  return 0;
}

/*
** Update an fts index.
*/
int sqlite4Fts5Update(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Description of fts index to update */
  int iRoot,
  Mem *pKey,                      /* Primary key blob */
  Mem *aArg,                      /* Array of arguments (see above) */
  int bDel,                       /* True for a delete, false for insert */
  char **pzErr                    /* OUT: Error message */
){
  int i;
  int rc = SQLITE4_OK;
  KVStore *pStore;
  TokenizeCtx sCtx;
  int nTnum = 0;
  u32 dummy = 0;

  u8 *aSpace = 0;
  int nSpace = 0;

  const u8 *pPK;
  int nPK;
  HashElem *pElem;

  if( iRoot==0 ) iRoot = pInfo->iRoot;
  pStore = db->aDb[pInfo->iDb].pKV;

  memset(&sCtx, 0, sizeof(sCtx));
  sCtx.db = db;
  sCtx.nCol = pInfo->nCol;
  sqlite4HashInit(db->pEnv, &sCtx.hash, 1);

  **   * space for the size record and key for this document, and
  **   * space for the updated global size record for the document set.
  **
  ** To make it easier, the below allocates enough space to simultaneously
  ** store the largest index record key and the largest possible global
  ** size record.
  */
  nSpace = (sqlite4VarintLen(iRoot) + 2 + sCtx.nMax + nPK) + 
           (9 * (2 + pInfo->nCol * sCtx.nStream));
  aSpace = sqlite4DbMallocRaw(db, nSpace);
  if( aSpace==0 ) rc = SQLITE4_NOMEM;

  for(pElem=sqliteHashFirst(&sCtx.hash); pElem; pElem=sqliteHashNext(pElem)){
    TokenizeTerm *pTerm = (TokenizeTerm *)sqliteHashData(pElem);
    if( rc==SQLITE4_OK ){
      int nToken = sqliteHashKeysize(pElem);
      char *zToken = (char *)sqliteHashKey(pElem);
      u8 *aKey = aSpace;
      int nKey;

      nKey = putVarint32(aKey, iRoot);
      aKey[nKey++] = 0x24;
      memcpy(&aKey[nKey], zToken, nToken);
      nKey += nToken;
      aKey[nKey++] = 0x00;
      memcpy(&aKey[nKey], pPK, nPK);
      nKey += nPK;

  }

  /* Write the size record into the db */
  if( rc==SQLITE4_OK ){
    u8 *aKey = aSpace;
    int nKey;

    nKey = putVarint32(aKey, iRoot);
    aKey[nKey++] = 0x00;
    memcpy(&aKey[nKey], pPK, nPK);
    nKey += nPK;

    if( bDel==0 ){
      Fts5Size sSz;
      sSz.nCol = pInfo->nCol;

  /* Update the global record */
  if( rc==SQLITE4_OK ){
    Fts5Size *pSz;                /* Deserialized global size record */
    i64 nRow;                     /* Number of rows in indexed table */
    u8 *aKey = aSpace;            /* Space to format the global record key */
    int nKey;                     /* Size of global record key in bytes */

    nKey = putVarint32(aKey, iRoot);
    aKey[nKey++] = 0x00;
    rc = fts5LoadSizeRecord(db, aKey, nKey, sCtx.nStream, pInfo, &nRow, &pSz);
    assert( rc!=SQLITE4_OK || pSz->nStream>=sCtx.nStream );

    if( rc==SQLITE4_OK ){
      int iCol;
      for(iCol=0; iCol<pSz->nCol; iCol++){

  return pInfo;
}

void sqlite4Fts5CodeUpdate(
  Parse *pParse, 
  Index *pIdx, 
  int iRegRoot,
  int iRegPk, 
  int iRegData,
  int bDel
){
  Vdbe *v;
  Fts5Info *pInfo;                /* p4 argument for FtsUpdate opcode */

  if( 0==(pInfo = fts5InfoCreate(pParse, pIdx, 0)) ) return;

  v = sqlite4GetVdbe(pParse);
  sqlite4VdbeAddOp3(v, OP_FtsUpdate, iRegPk, iRegRoot, iRegData);
  sqlite4VdbeChangeP4(v, -1, (const char *)pInfo, P4_FTS5INFO);
  sqlite4VdbeChangeP5(v, (u8)bDel);
}

void sqlite4Fts5CodeQuery(
  Parse *pParse,
  Index *pIdx,

  int *pnMatch,
  int *pnDoc,
  int *pnRelevant
){
  return SQLITE4_OK;
}

static void fts5StrLoadRowcounts(
  Fts5Str *pStr, 
  int nStream, 
  int *anRow,
  int *anRowC,
  int *anRowS,
  int *pnRowCS
){
  u32 mask = 0;
  int iPrevCol = -1;
  InstanceList sList;

  fts5InstanceListInit(pStr->aList, pStr->nList, &sList);
  while( 0==fts5InstanceListNext(&sList) ){
    if( iPrevCol<0 ) (*pnRowCS)++;
    if( sList.iCol!=iPrevCol ){
      mask = 0;
      anRowC[sList.iCol]++;
    }
    if( (mask & (1<<sList.iStream))==0 ){
      anRow[sList.iCol * nStream + sList.iStream]++;
      mask |= (1<<sList.iStream);
      iPrevCol = sList.iCol;
    }
  }
}

static int fts5ExprLoadRowcounts(
  sqlite4 *db, 
  Fts5Cursor *pCsr,
  int nStream,
  Fts5ExprNode *pNode, 
  int *piStr
){
  int rc = SQLITE4_OK;

  if( pNode ){
    Fts5Info *pInfo = pCsr->pInfo;

    if( pNode->eType==TOKEN_PRIMITIVE ){

      Fts5Phrase *pPhrase = pNode->pPhrase;
      int iStr = *piStr;

      rc = fts5ExprAdvance(db, pNode, 1);
      while( rc==SQLITE4_OK && pNode->aPk ){
        int nIncr =  pInfo->nCol * nStream;      /* Values for each Fts5Str */
        int i;
        for(i=0; i<pPhrase->nStr; i++){
          int *anRow = &pCsr->anRow[(iStr+i) * pInfo->nCol * nStream];
          int *anRowC = &pCsr->anRowC[(iStr+i) * pInfo->nCol];
          int *anRowS = &pCsr->anRowS[(iStr+i) * nStream];
          int *pnRowCS = &pCsr->anRowCS[iStr+i];
          fts5StrLoadRowcounts(
              &pPhrase->aStr[i], nStream, anRow, anRowC, anRowS, pnRowCS
          );
        }
        rc = fts5ExprAdvance(db, pNode, 0);
      }

      *piStr = iStr + pPhrase->nStr;
    }

    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pLeft, piStr);
    }
    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pNode->pRight, piStr);
    }
  }

  return rc;
}

static int fts5CsrLoadRowcounts(Fts5Cursor *pCsr){
  int rc = SQLITE4_OK;

  if( pCsr->anRow==0 ){
    int nStream = pCsr->pGlobal->nStream;
    sqlite4 *db = pCsr->db;
    Fts5Expr *pCopy;
    Fts5Expr *pExpr = pCsr->pExpr;
    Fts5Info *pInfo = pCsr->pInfo;
    int *anRow;
    int iPhrase = 0;

    pCsr->anRow = anRow = (int *)sqlite4DbMallocZero(db, sizeof(int) * (
        pExpr->nPhrase * pInfo->nCol * pCsr->pGlobal->nStream
      + pExpr->nPhrase * pInfo->nCol
      + pExpr->nPhrase * pCsr->pGlobal->nStream
      + pExpr->nPhrase 
    ));
    if( !anRow ) return SQLITE4_NOMEM;
    pCsr->anRowC = &anRow[pExpr->nPhrase*pInfo->nCol*pCsr->pGlobal->nStream];
    pCsr->anRowS = &pCsr->anRowC[pExpr->nPhrase * pInfo->nCol];
    pCsr->anRowCS = &pCsr->anRowS[pExpr->nPhrase * pCsr->pGlobal->nStream];

    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->iRoot, pInfo->azCol, pInfo->nCol, pCsr->zExpr, &pCopy, 0
    );
    if( rc==SQLITE4_OK ){
      rc = fts5OpenExprCursors(db, pInfo, pCopy->pRoot);
    }
    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pCsr, nStream, pCopy->pRoot, &iPhrase);
    }

    fts5ExpressionFree(db, pCopy);
  }

  return rc;
}

      int i;
      int nRow = 0;
      int nStream = pCsr->pGlobal->nStream;
      int nCol = pCsr->pInfo->nCol;
      int *aRow = &pCsr->anRow[iP * nStream * nCol];

      if( iC<0 && iS<0 ){

        nRow = pCsr->anRowCS[iP];
      }else if( iC<0 ){
        for(i=0; i<nCol; i++) nRow += aRow[i*nStream + iS];
      }else if( iS<0 ){
        nRow = pCsr->anRowC[iP*nCol + iC];
      }else if( iC<nCol && iS<nStream ){
        nRow = aRow[iC * nStream + iS];
      }

      *pn = nRow;
    }
  }

** on the nature of both the documents and queries. The implementation
** below sets each parameter to the midpoint of the suggested range.
*/
static void fts5Rank(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){
  const double b = 0.65;
  const double k1 = 1.6;

  sqlite4 *db = sqlite4_context_db_handle(pCtx);
  int rc = SQLITE4_OK;            /* Error code */
  Fts5RankCtx *p;                 /* Structure to store reusable values */
  int i;                          /* Used to iterate through phrases */
  double rank = 0.0;              /* UDF return value */

  int bExplain = 0;
  char *zExplain = 0;

  if( sqlite4_user_data(pCtx) ) bExplain = 1;

  p = sqlite4_get_auxdata(pCtx, 0);
  if( p==0 ){

    int nPhrase;                  /* Number of phrases in query expression */
    int nByte;                    /* Number of bytes of data to allocate */

    sqlite4_mi_phrase_count(pCtx, &nPhrase);
    nByte = sizeof(Fts5RankCtx) + nPhrase * sizeof(double);
    p = (Fts5RankCtx *)sqlite4DbMallocZero(db, nByte);
    sqlite4_set_auxdata(pCtx, 0, (void *)p, fts5RankFreeCtx);

      p->aIdf = (double *)&p[1];

      /* Determine the IDF weight for each phrase in the query. */
      rc = sqlite4_mi_total_rows(pCtx, &N);
      for(i=0; rc==SQLITE4_OK && i<nPhrase; i++){
        rc = sqlite4_mi_row_count(pCtx, -1, -1, i, &ni);
        if( rc==SQLITE4_OK ){
          assert( ni<=N );
          p->aIdf[i] = log((0.5 + N - ni) / (0.5 + ni));
        }
      }

      /* Determine the average document length */
      if( rc==SQLITE4_OK ){
        int nTotal;

    ** in this row (within any column). And dl to the number of tokens in
    ** the current row (again, in any column).  */
    rc = sqlite4_mi_match_count(pCtx, -1, -1, i, &tf); 
    if( rc==SQLITE4_OK ) rc = sqlite4_mi_size(pCtx, -1, -1, &dl); 

    /* Calculate the normalized document length */
    L = (double)dl / p->avgdl;


    /* Calculate the contribution to the rank made by this phrase. Then
    ** add it to variable rank.  */
    prank = (p->aIdf[i] * tf) / (k1 * ( (1.0 - b) + b * L) + tf);
    rank += prank;

    if( bExplain ){
      zExplain = sqlite4MAppendf(
          db, zExplain, "%s(idf=%.2f L=%.2f tf=%d) rank=%.2f", zExplain,
          p->aIdf[i], L, tf, prank
      );
      if( (i+1)<p->nPhrase ){
        zExplain = sqlite4MAppendf(db, zExplain, "%s<br>", zExplain);
      }
    }
  }

  if( rc==SQLITE4_OK ){
    if( bExplain ){
      if( p->nPhrase>1 ){
        zExplain = sqlite4MAppendf(
            db, zExplain, "%s<br>total=%.2f", zExplain, rank
        );
      }
      sqlite4_result_text(pCtx, zExplain, -1, SQLITE4_TRANSIENT);
      sqlite4DbFree(db, zExplain);
    }else{
      sqlite4_result_double(pCtx, rank);
    }
  }else{
    sqlite4_result_error_code(pCtx, rc);
  }
}

typedef struct Snippet Snippet;
typedef struct SnippetText SnippetText;

){
  SnippetCtx *p = (SnippetCtx *)pCtx;

  if( iOff<p->iOff ){
    return 0;
  }else if( iOff>=(p->iOff + p->nToken) ){
    fts5SnippetAppend(p, &p->zText[p->iFrom], p->iTo - p->iFrom);
    fts5SnippetAppend(p, p->zEllipses, -1);
    p->iFrom = -1;
    return 1;
  }else{
    int bHighlight;               /* True to highlight term */

    bHighlight = (p->mask & ((u64)1 << (iOff-p->iOff))) ? 1 : 0;

    if( p->iFrom==0 && p->iOff!=0 ){
      p->iFrom = iSrc;
      if( p->pOut->nOut==0 ) fts5SnippetAppend(p, p->zEllipses, -1);
    }

    if( bHighlight ){

      int nScore = 0;

      int nPTok;
      int iPTok;

      if( iColumn>=0 && iColumn!=iCol ) continue;

      allmask |= ((u64)1 << iPhrase);

      nShift = ((iPrevCol==iCol) ? (iOff-iPrev) : 100);

      for(iMask=0; iMask<nPhrase; iMask++){
        if( nShift<64){
          aMask[iMask] = aMask[iMask] >> nShift;
        }else{
          aMask[iMask] = 0;
        }
      }
      sqlite4_mi_phrase_token_count(pCtx, iPhrase, &nPTok);
      for(iPTok=0; iPTok<nPTok; iPTok++){
        aMask[iPhrase] = aMask[iPhrase] | ((u64)1 << (nToken-1+iPTok));
      }

      for(iMask=0; iMask<nPhrase; iMask++){
        int iBit;
        if( aMask[iMask] ){
          nScore += ((((u64)1 << iMask) & mask) ? 100 : 1);
        }else{
          miss |= ((u64)1 << iMask);
        }
        tmask = tmask | aMask[iMask];
        /* TODO: This is the Hamming Weight. There are much more efficient
        ** ways to calculate it. */
        for(iBit=0; iBit<nToken; iBit++){
          if( tmask & ((u64)1 << iBit) ) nScore++;
        }
      }

      if( nScore>nBest ){
        hlmask = tmask;
        missmask = miss;
        nBest = nScore;
        iBestOff = iOff;

  int nLead = 0;
  int nShift = 0;

  u64 mask = pSnip->hlmask;
  int iOff = pSnip->iOff;

  if( mask==0 ) return;
  assert( mask & ((u64)1 << (nToken-1)) );

  for(i=0; (mask & ((u64)1 << i))==0; i++);
  nLead = i;

  nShift = (nLead/2);
  if( iOff+nShift > nSz-nToken ) nShift = (nSz-nToken) - iOff;
  if( iOff+nShift < 0 ) nShift = -1 * iOff;

  iOff += nShift;

  rc = sqlite4_create_tokenizer(db, "simple", (void *)pEnv, 
      fts5SimpleCreate, fts5SimpleTokenize, fts5SimpleDestroy
  );
  if( rc!=SQLITE4_OK ) return rc;

  rc = sqlite4_create_mi_function(db, "rank", 0, SQLITE4_UTF8, 0, fts5Rank, 0);
  if( rc!=SQLITE4_OK ) return rc;
  rc = sqlite4_create_mi_function(
      db, "erank", 0, SQLITE4_UTF8, (void *)1, fts5Rank, 0
  );
  if( rc!=SQLITE4_OK ) return rc;

  rc = sqlite4_create_mi_function(
      db, "snippet", -1, SQLITE4_UTF8, 0, fts5Snippet, 0
  );
  return rc;
}

  /* Write the entry to each index. */
  for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){
    assert( pIdx->eIndexType!=SQLITE4_INDEX_PRIMARYKEY || aRegIdx[i] );
    if( pIdx->eIndexType==SQLITE4_INDEX_FTS5 ){
      int iPK;
      sqlite4FindPrimaryKey(pTab, &iPK);
      sqlite4Fts5CodeUpdate(pParse, pIdx, 0, aRegIdx[iPK], regContent, 0);
    }
    else if( aRegIdx[i] ){
      int regData = 0;
      int flags = 0;
      if( pIdx->eIndexType==SQLITE4_INDEX_PRIMARYKEY ){
        regData = regRec;
        flags = pik_flags;

  memset(p, 0, sizeof(RowSet));
  p->isSorted = 1;
}


static u8 *rowsetAllocateChunk(RowSet *p, int nByte){
  int rowChunkSize = ROUND8(sizeof(RowSetChunk));
  RowSetChunk *pNew;              /* New RowSetChunk */
  int nAlloc;                     /* Bytes to request from malloc() */
  nAlloc =  rowChunkSize + nByte;
  pNew = (RowSetChunk *)sqlite4DbMallocRaw(p->db, nAlloc);
  if( !pNew ) return 0;
  pNew->pNextChunk = p->pChunk;
  p->pChunk = pNew;
  return (u8 *)(&pNew[1]);
}

static int rowsetEntryKeyCmp(RowSetEntry *pLeft, const u8 *aKey, int nKey){
  int nCmp = SQLITE4_MIN(pLeft->nKey, nKey);
  int res;
  res = memcmp(pLeft->aKey, aKey, nCmp);
  return (res ? res : (pLeft->nKey - nKey));

void sqlite4ShutdownFts5(sqlite4 *db);
void sqlite4CreateUsingIndex(Parse*, CreateIndex*, ExprList*, Token*, Token*);

int sqlite4Fts5IndexSz(void);
void sqlite4Fts5IndexInit(Parse *, Index *, ExprList *);
void sqlite4Fts5IndexFree(sqlite4 *, Index *);

int sqlite4Fts5Update(sqlite4 *, Fts5Info *, int, Mem *, Mem *, int, char **);
void sqlite4Fts5FreeInfo(sqlite4 *db, Fts5Info *);
void sqlite4Fts5CodeUpdate(Parse *, Index *pIdx, int, int, int, int);
void sqlite4Fts5CodeCksum(Parse *, Index *, int, int, int);
void sqlite4Fts5CodeQuery(Parse *, Index *, int, int, int);

int sqlite4Fts5Pk(Fts5Cursor *, int, KVByteArray **, KVSize *);
int sqlite4Fts5Next(Fts5Cursor *pCsr);

int sqlite4Fts5EntryCksum(sqlite4 *, Fts5Info *, Mem *, Mem *, i64 *);

    sqlite4DebugPrintf("SQL-trace: %s\n", zTrace);
  }
#endif /* SQLITE4_DEBUG */
  break;
}
#endif

/* Opcode: FtsUpdate P1 P2 P3 P4 P5
**
** This opcode is used to write to an FTS index. P4 points to an Fts5Info 
** object describing the index.
**
** If argument P5 is non-zero, then entries are removed from the FTS index.
** If it is zero, then entries are inserted. In other words, when a row
** is deleted from a table with an FTS index, this opcode is invoked with
** P5==1. When a row is inserted, it is invoked with P5==0. If an existing
** row is updated, this opcode is invoked twice - once with P5==1 and then
** again with P5==0.
**
** Register P1 contains the PK (a blob in key format) of the affected row.
** P3 is the first in an array of N registers, where N is the number of
** columns in the indexed table. Each register contains the value for the
** corresponding table column.
**
** If P2 is non-zero, then it is a register containing the root page number
** of the fts index to update. If it is zero, then the root page of the 
** index is available as part of the Fts5Info structure.
*/
case OP_FtsUpdate: {
  Fts5Info *pInfo;                /* Description of fts5 index to update */
  Mem *pKey;                      /* Primary key of indexed row */
  Mem *aArg;                      /* Pointer to array of N arguments */
  Mem *pRoot;                     /* Root page number */
  int iRoot;

  assert( pOp->p4type==P4_FTS5INFO );
  pInfo = pOp->p4.pFtsInfo;
  aArg = &aMem[pOp->p3];
  pKey = &aMem[pOp->p1];

  if( pOp->p2 ){
    iRoot = aMem[pOp->p2].u.i;
  }else{
    iRoot = 0;
  }

  rc = sqlite4Fts5Update(db, pInfo, iRoot, pKey, aArg, pOp->p5, &p->zErrMsg);
  break;
}

/*
** Opcode: FtsCksum P1 * P3 P4 P5
**
** This opcode is used by the integrity-check procedure that verifies that

  CREATE INDEX ft ON t2 USING fts5(tukenizer=simple);
} {1 {unrecognized argument: "tukenizer"}}

do_catchsql_test 2.3 { 
  CREATE INDEX ft ON t2 USING fts5("a b c");
} {1 {unrecognized argument: "a b c"}}


do_catchsql_test 2.4 {
  CREATE INDEX ft ON t2 USING fts5(tokenizer="nosuch");
} {1 {no such tokenizer: "nosuch"}}

#-------------------------------------------------------------------------
#
reset_db

do_execsql_test 3.1 {
  CREATE TABLE t1(a, b, c, PRIMARY KEY(a));
  INSERT INTO t1 VALUES(1, 'a b c d', 'e f g h');
  INSERT INTO t1 VALUES(2, 'e f g h', 'a b c d');
}

do_execsql_test 3.2 {
  CREATE INDEX ft ON t1 USING fts5();
  PRAGMA fts_check(ft);
} {ok}


finish_test

based system. Additionally, avoiding random writes in favour of largely
contiguous updates (as LSM does) can significantly reduce the wear on SSD or
flash memory devices.

<p>Although it has quite different features to LSM in other respects, 
LevelDB makes similar performance tradeoffs.

<p>Benchmark test results for LSM are <a href=lsmperf.wiki>available here</a>.



<h1 id=using_lsm_in_applications>2. Using LSM in Applications </h1>

<p>LSM is not currently built or distributed independently. Instead, it
is part of the SQLite4 library. To use LSM in an application, the application
links against libsqlite4 and includes the header file "lsm.h" in any files
that access the LSM API.