SQLite4  Check-in [8d94102cd3]

/*
** Structure types used by this module.
*/
typedef struct Fts5Expr Fts5Expr;
typedef struct Fts5ExprNode Fts5ExprNode;
typedef struct Fts5List Fts5List;
typedef struct Fts5MatchIter Fts5MatchIter;
typedef struct Fts5Parser Fts5Parser;
typedef struct Fts5ParserToken Fts5ParserToken;
typedef struct Fts5Phrase Fts5Phrase;
typedef struct Fts5Prefix Fts5Prefix;
typedef struct Fts5Size Fts5Size;
typedef struct Fts5Str Fts5Str;
typedef struct Fts5Token Fts5Token;

  Fts5ExprNode *pLeft;
  Fts5ExprNode *pRight;
  const u8 *aPk;                  /* Primary key of current entry (or null) */
  int nPk;                        /* Size of aPk[] in bytes */
};

struct Fts5Expr {
  Fts5ExprNode *pRoot;            /* Root node of expression */

  int nPhrase;                    /* Number of Fts5Str objects in query */
  Fts5Str **apPhrase;             /* All Fts5Str objects */
};

/*
** FTS5 specific cursor data.
*/
struct Fts5Cursor {
  sqlite4 *db;
  Fts5Info *pInfo;
  Fts5Expr *pExpr;                /* MATCH expression for this cursor */
  char *zExpr;                    /* Full text of MATCH expression */
  KVByteArray *aKey;              /* Buffer for primary key */
  int nKeyAlloc;                  /* Bytes allocated at aKey[] */

  KVCursor *pCsr;                 /* Cursor used to retrive values */
  Mem *aMem;                      /* Array of column values */
  int bMemValid;                  /* True if contents of aMem[] are valid */

  Fts5Size *pSz;                  /* Local size data */
  Fts5Size *pGlobal;              /* Global size data */
  i64 nGlobal;                    /* Total number of rows in table */
  int *anRow;



  Fts5MatchIter *pIter;           /* Used by mi_match_detail() */




};

/*
** A deserialized 'size record' (see above).
*/
struct Fts5Size {
  int nCol;                       /* Number of columns in indexed table */

  int iList;

  /* The current entry */
  int iCol;
  int iStream;
  int iOff;
};

/*
** An instance of this structure is used by the sqlite4_mi_match_detail()
** API to iterate through matches. 
*/
struct Fts5MatchIter {
  int bValid;                     /* True if aList[] is current row */
  int iCurrent;                   /* Current index in aList[] (or -1) */
  int iMatch;                     /* Current iMatch value */
  InstanceList *aList;            /* One iterator for each phrase in expr */
};

/*
** Return true for EOF, or false if the next entry is valid.
*/
static int fts5InstanceListNext(InstanceList *p){
  int i = p->iList;
  int bRet = 1;

    sqlite4DbFree(db, pNode);
  }
}

static void fts5ExpressionFree(sqlite4 *db, Fts5Expr *pExpr){
  if( pExpr ){
    fts5FreeExprNode(db, pExpr->pRoot);
    sqlite4DbFree(db, pExpr->apPhrase);
    sqlite4DbFree(db, pExpr);
  }
}

typedef struct ExprHier ExprHier;
struct ExprHier {
  Fts5ExprNode **ppNode;

  *pp = pNode;
  (*paHier)[*pnHier].ppNode = &pNode->pRight;
  (*paHier)[*pnHier].nOpen = 0;
  (*pnHier)++;

  return SQLITE4_OK;
}

static void fts5FindStrings(Fts5ExprNode *p, Fts5Str ***papStr){
  if( p ){
    if( p->eType==TOKEN_PRIMITIVE ){
      int i;
      Fts5Str *aStr = p->pPhrase->aStr;
      for(i=0; i<p->pPhrase->nStr; i++){
        **papStr = &aStr[i];
        (*papStr)++;
      }
    }
    fts5FindStrings(p->pLeft, papStr);
    fts5FindStrings(p->pRight, papStr);
  }
}

static int fts5ParseExpression(
  sqlite4 *db,                    /* Database handle */
  Fts5Tokenizer *pTokenizer,      /* Tokenizer module */
  sqlite4_tokenizer *p,           /* Tokenizer instance */
  int iRoot,                      /* Root page number of FTS index */
  char **azCol,                   /* Array of column names (nul-term'd) */

  if( rc==SQLITE4_OK && *aHier[nHier-1].ppNode==0 ){
    rc = SQLITE4_ERROR;
  }
  for(i=0; rc==SQLITE4_OK && i<nHier; i++){
    if( aHier[i].nOpen>0 ) rc = SQLITE4_ERROR;
  }

  if( rc==SQLITE4_OK ){
    pExpr->nPhrase = nStr;
    pExpr->apPhrase = (Fts5Str**)sqlite4DbMallocZero(db, sizeof(Fts5Str*)*nStr);
    if( pExpr->apPhrase==0 ){
      rc = SQLITE4_NOMEM;
    }else{
      Fts5Str **a = pExpr->apPhrase;
      fts5FindStrings(pExpr->pRoot, &a);
    }
  }

  if( rc!=SQLITE4_OK ){
    fts5ExpressionFree(db, pExpr);
    *pzErr = sParse.zErr;


    pExpr = 0;
  }
  *ppExpr = pExpr;
  sqlite4DbFree(db, aHier);
  return rc;
}

/*
** Search for the Fts5Tokenizer object named zName. Return a pointer to it
** if it exists, or NULL otherwise.

    nByte += nCol * sizeof(char *);
  }

  pInfo = sqlite4DbMallocZero(db, nByte);
  if( pInfo ){
    pInfo->iDb = sqlite4SchemaToIndex(db, pIdx->pSchema);
    pInfo->iRoot = pIdx->tnum;
    sqlite4FindPrimaryKey(pIdx->pTable, &pInfo->iTbl);
    pInfo->nCol = pIdx->pTable->nCol;
    fts5TokenizerCreate(pParse, pIdx->pFts, &pInfo->pTokenizer, &pInfo->p);

    if( pInfo->p==0 ){
      assert( pParse->nErr );
      sqlite4DbFree(db, pInfo);
      pInfo = 0;

static int fts5OpenCursors(sqlite4 *db, Fts5Info *pInfo, Fts5Cursor *pCsr){
  return fts5OpenExprCursors(db, pInfo, pCsr->pExpr->pRoot);
}

void sqlite4Fts5Close(sqlite4 *db, Fts5Cursor *pCsr){
  if( pCsr ){
    fts5ExpressionFree(db, pCsr->pExpr);
    sqlite4DbFree(db, pCsr->pIter);
    sqlite4DbFree(db, pCsr->aKey);
    sqlite4DbFree(db, pCsr->anRow);
    sqlite4DbFree(db, pCsr);
  }
}

static int fts5TokenAdvanceToMatch(

  assert( rc!=SQLITE4_NOTFOUND );
  return rc;
}

int sqlite4Fts5Next(Fts5Cursor *pCsr){
  sqlite4DbFree(pCsr->db, pCsr->pSz);
  pCsr->pSz = 0;
  pCsr->bMemValid = 0;
  return fts5ExprAdvance(pCsr->db, pCsr->pExpr->pRoot, 0);
}

int sqlite4Fts5Open(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Index description */
  const char *zMatch,             /* Match expression */

*/
int sqlite4Fts5Valid(Fts5Cursor *pCsr){
  return( pCsr->pExpr->pRoot->aPk!=0 );
}

int sqlite4Fts5Pk(
  Fts5Cursor *pCsr, 
  int iTbl,
  KVByteArray **paKey,
  KVSize *pnKey
){
  int i;
  int nReq;
  const u8 *aPk;
  int nPk;

  }else{
    rc = fts5CsrLoadGlobal(pCsr);
    if( rc==SQLITE4_OK ) *pn = pCsr->nGlobal;
  }
  return rc;
}


int sqlite4_mi_column_value(
  sqlite4_context *pCtx, 
  int iCol, 
  sqlite4_value **ppVal
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;
  if( pCsr==0 ){
    rc = SQLITE4_MISUSE;
  }else{
    if( pCsr->bMemValid==0 ){
      sqlite4 *db = pCsr->db;

      Fts5Info *pInfo = pCsr->pInfo;
      if( pCsr->aMem==0 ){
        int nByte = sizeof(Mem) * pInfo->nCol;
        pCsr->aMem = (Mem *)sqlite4DbMallocZero(db, nByte);
        if( pCsr->aMem==0 ){
          rc = SQLITE4_NOMEM;
        }else{
          int i;
          for(i=0; i<pInfo->nCol; i++){
            pCsr->aMem[i].db = db;
          }
        }
      }

      if( pCsr->pCsr==0 && rc==SQLITE4_OK ){
        KVStore *pStore = db->aDb[pInfo->iDb].pKV;
        rc = sqlite4KVStoreOpenCursor(pStore, &pCsr->pCsr);
      }

      if( rc==SQLITE4_OK ){
        u8 *aKey = 0; int nKey;     /* Primary key for current row */
        const u8 *aData; int nData; /* Data record for current row */

        rc = sqlite4Fts5Pk(pCsr, pInfo->iTbl, &aKey, &nKey);
        if( rc==SQLITE4_OK ){
          rc = sqlite4KVCursorSeek(pCsr->pCsr, aKey, nKey, 0);
          if( rc==SQLITE4_NOTFOUND ){
            rc = SQLITE4_CORRUPT_BKPT;
          }
        }

        if( rc==SQLITE4_OK ){
          rc = sqlite4KVCursorData(pCsr->pCsr, 0, -1, &aData, &nData);
        }

        if( rc==SQLITE4_OK ){
          int i;
          ValueDecoder *pCodec;   /* The decoder object */

          rc = sqlite4VdbeCreateDecoder(db, aData, nData, pInfo->nCol, &pCodec);
          for(i=0; rc==SQLITE4_OK && i<pInfo->nCol; i++){
            rc = sqlite4VdbeDecodeValue(pCodec, i, 0, &pCsr->aMem[i]);
          }
          sqlite4VdbeDestroyDecoder(pCodec);
        }

        if( rc==SQLITE4_OK ) pCsr->bMemValid = 1;
      }
    }

    if( rc==SQLITE4_OK ){
      assert( pCsr->bMemValid );
      *ppVal = &pCsr->aMem[iCol];
    }
  }

  return rc;
}

int sqlite4_mi_tokenize(
  sqlite4_context *pCtx,
  const char *zText,
  int nText,
  void *p,
  int(*x)(void *, int, int, const char *, int, int, int)
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;

  if( pCsr==0 ){
    rc = SQLITE4_MISUSE;
  }else{
    Fts5Info *pInfo = pCsr->pInfo;
    rc = pInfo->pTokenizer->xTokenize(p, pInfo->p, zText, nText, x);
  }
  return rc;
}

static Fts5Str *fts5FindStr(Fts5ExprNode *p, int *piStr){
  Fts5Str *pRet = 0;
  if( p->eType==TOKEN_PRIMITIVE ){

        for(i=0; i<nStream; i++) nRow += aRow[nStream*iC + iS];
      }else if( iC<nCol && iS<nStream ){
        nRow = aRow[iC * nStream + iS];
      }

      *pn = nRow;
    }
  }
  return rc;
}

static void fts5IterSetCurrent(Fts5MatchIter *pIter, int nList){
  InstanceList *pBest = 0;
  int i;

  for(i=0; i<nList; i++){
    InstanceList *p = &pIter->aList[i];
    if( fts5InstanceListEof(p)==0 ){
      if( (pBest==0)
       || (p->iCol<pBest->iCol)
       || (p->iCol==pBest->iCol && p->iOff<pBest->iOff)
      ){
        pBest = p;
      }
    }
  }

  if( pBest==0 ){
    pIter->iCurrent = -1;
  }else{
    pIter->iCurrent = pBest - pIter->aList;
  }
}

int sqlite4_mi_match_detail(
  sqlite4_context *pCtx,          /* Context object passed to mi function */
  int iMatch,                     /* Index of match */
  int *piOff,                     /* OUT: Token offset of match */
  int *piC,                       /* OUT: Column number of match iMatch */
  int *piS,                       /* OUT: Stream number of match iMatch */
  int *piP                        /* OUT: Phrase number of match iMatch */
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;
  if( pCsr==0 ){
    rc = SQLITE4_MISUSE;
  }else{
    int nPhrase = pCsr->pExpr->nPhrase;
    Fts5MatchIter *pIter = pCsr->pIter;
    if( pIter==0 ){
      pCsr->pIter = pIter = (Fts5MatchIter *)sqlite4DbMallocZero(
          pCsr->db, sizeof(Fts5MatchIter) + sizeof(InstanceList)*nPhrase
      );
      if( pIter ){
        pIter->aList = (InstanceList *)&pIter[1];
      }else{
        rc = SQLITE4_NOMEM;
      }
    }

    if( rc==SQLITE4_OK && (pIter->bValid==0 || iMatch<pIter->iMatch) ){
      int i;
      for(i=0; i<pCsr->pExpr->nPhrase; i++){
        Fts5Str *pStr = pCsr->pExpr->apPhrase[i];
        fts5InstanceListInit(pStr->aList, pStr->nList, &pIter->aList[i]);
        fts5InstanceListNext(&pIter->aList[i]);
      }

      pIter->iMatch = 0;
      fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase);
    }

    if( rc==SQLITE4_OK ){
      assert( pIter->iMatch<=iMatch );
      while( pIter->iCurrent>=0 && pIter->iMatch<iMatch ){
        fts5InstanceListNext(&pIter->aList[pIter->iCurrent]);
        fts5IterSetCurrent(pIter, pCsr->pExpr->nPhrase);
      }
      if( pIter->iCurrent<0 ){
        rc = SQLITE4_NOTFOUND;
      }else{
        InstanceList *p = &pIter->aList[pIter->iCurrent];
        *piOff = p->iOff;
        *piC = p->iCol;
        *piS = p->iStream;
        *piP = pIter->iCurrent;
      }
    }
  }
  return rc;
}

/**************************************************************************
***************************************************************************
** Below this point is test code.

  if( rc==SQLITE4_OK ){
    sqlite4_result_double(pCtx, rank);
  }else{
    sqlite4_result_error_code(pCtx, rc);
  }
}

typedef struct SnippetCtx SnippetCtx;
struct SnippetCtx {
  sqlite4 *db;                    /* Database handle */
  int nToken;                     /* Number of tokens in snippet */
  int iOff;                       /* First token in snippet */
  u64 mask;                       /* Snippet mask. Highlight these terms */

  char *zOut;                     /* Pointer to snippet text */
  int nOut;                       /* Size of zOut in bytes */
  int nAlloc;                     /* Bytes of space allocated at zOut */

  int iFrom;
  const char *zText;              /* Document to extract snippet from */

  int rc;                         /* Set to NOMEM if OOM is encountered */
};

static void fts5SnippetAppend(SnippetCtx *p, const char *z, int n){
  if( p->rc==SQLITE4_OK ){
    if( (p->nOut + n) > p->nAlloc ){
      int nNew = (p->nOut+n) * 2;

      p->zOut = sqlite4DbReallocOrFree(p->db, p->zOut, nNew);
      if( p->zOut==0 ){
        p->rc = SQLITE4_NOMEM;
        return;
      }
      p->nAlloc = sqlite4DbMallocSize(p->db, p->zOut);
    }

    memcpy(&p->zOut[p->nOut], z, n);
    p->nOut += n;
  }
}

static int fts5SnippetCb(
  void *pCtx, 
  int iStream, 
  int iOff, 
  const char *z, int n,
  int iSrc, int nSrc
){
  SnippetCtx *p = (SnippetCtx *)pCtx;

  if( iOff<p->iOff ){
    return 0;
  }else if( iOff>(p->iOff + p->nToken) ){
    fts5SnippetAppend(p, &p->zText[p->iFrom], iSrc - p->iFrom);
    fts5SnippetAppend(p, "...", 3);
    p->iFrom = -1;
    return 1;
  }else{
    int bHighlight;               /* True to highlight term */

    bHighlight = (p->mask & (1 << (p->iOff+p->nToken - iOff - 1))) ? 1 : 0;

    if( p->iFrom==0 && p->iOff!=0 ){
      p->iFrom = iSrc;
      fts5SnippetAppend(p, "...", 3);
    }

    if( bHighlight ){
      fts5SnippetAppend(p, &p->zText[p->iFrom], iSrc - p->iFrom);
      fts5SnippetAppend(p, "[", 1);
      fts5SnippetAppend(p, &p->zText[iSrc], nSrc);
      fts5SnippetAppend(p, "]", 1);
      p->iFrom = iSrc+nSrc;
    }
  }

  return 0;
}

static int fts5SnippetText(
  sqlite4_context *pCtx, 
  int iCol,
  int iOff,
  int nToken,
  u64 mask
){
  int rc;
  sqlite4_value *pVal = 0;

  rc = sqlite4_mi_column_value(pCtx, iCol, &pVal);
  if( rc==SQLITE4_OK ){
    SnippetCtx sCtx;
    int nText;

    nText = sqlite4_value_bytes(pVal);
    memset(&sCtx, 0, sizeof(sCtx));
    sCtx.zText = (const char *)sqlite4_value_text(pVal);
    sCtx.db = sqlite4_context_db_handle(pCtx);
    sCtx.nToken = nToken;
    sCtx.iOff = iOff;
    sCtx.mask = mask;

    sqlite4_mi_tokenize(pCtx, sCtx.zText, nText, &sCtx, fts5SnippetCb);
    if( sCtx.rc==SQLITE4_OK && sCtx.iFrom>0 ){
      fts5SnippetAppend(&sCtx, &sCtx.zText[sCtx.iFrom], nText - sCtx.iFrom);
    }
    rc = sCtx.rc;

    sqlite4_result_text(pCtx, sCtx.zOut, sCtx.nOut, SQLITE4_TRANSIENT);
    sqlite4DbFree(sCtx.db, sCtx.zOut);
  }

  return rc;
}

static int fts5BestSnippet(
  sqlite4_context *pCtx, 
  u64 mask,                       /* Mask of high-priority phrases */
  int nToken,
  int *piOff,
  int *piCol,
  u64 *pMask
){
  sqlite4 *db = sqlite4_context_db_handle(pCtx);
  int nPhrase;
  int rc = SQLITE4_OK;
  int i;
  int iPrev = 0;
  int iPrevCol = 0;
  u64 *aMask;
  u64 lmask = (((u64)1) << nToken) - 1;

  int iBestOff = 0;
  int iBestCol = 0;
  int nBest = 0;
  u64 bmask = 0;

  sqlite4_mi_phrase_count(pCtx, &nPhrase);
  aMask = sqlite4DbMallocZero(db, sizeof(u64) * nPhrase);
  if( !aMask ) return SQLITE4_NOMEM;

  /* Iterate through all matches for all phrases */
  for(i=0; rc==SQLITE4_OK; i++){
    int iOff;
    int iCol;
    int iStream;
    int iPhrase;
    u64 tmask = 0;

    rc = sqlite4_mi_match_detail(pCtx, i, &iOff, &iCol, &iStream, &iPhrase);
    if( rc==SQLITE4_OK ){
      int iMask;
      int nShift; 
      int nScore = 0;

      nShift = ((iPrevCol==iCol) ? (iOff-iPrev) : 100);

      for(iMask=0; iMask<nPhrase; iMask++){
        if( nShift<64){
          aMask[iMask] = aMask[iMask] << nShift;
        }else{
          aMask[iMask] = 0;
        }
      }
      aMask[iPhrase] = aMask[iMask] | 0x0001;

      for(iMask=0; iMask<nPhrase; iMask++){
        if( (aMask[iMask] & lmask) ){
          nScore += ((aMask[iMask] & mask) ? 100 : 1);
        }
        tmask = tmask | aMask[iMask];
      }

      if( nScore>nBest ){
        bmask = (tmask & lmask);
        nBest = nScore;
        iBestOff = iOff;
        iBestCol = iCol;
      }

      iPrev = iOff;
      iPrevCol = iCol;
    }
  }

  *piOff = iBestOff;
  *piCol = iBestCol;
  *pMask = bmask;

  sqlite4DbFree(db, aMask);
  return rc;
}

static void fts5Snippet(sqlite4_context *pCtx, int nArg, sqlite4_value **apArg){
  int nToken = 15;
  u64 hlmask = 0;
  u64 mask = 0;
  int iOff = 0;
  int iCol = 0;
  int rc;

  rc = fts5BestSnippet(pCtx, mask, nToken, &iOff, &iCol, &hlmask);
  if( rc==SQLITE4_OK ){
    rc = fts5SnippetText(pCtx, iCol, iOff, nToken, hlmask);
  }
  if( rc!=SQLITE4_OK ){
    sqlite4_result_error_code(pCtx, rc);
  }
}

static int fts5SimpleTokenize(

  void *pCtx, sqlite4_tokenizer *p,
  const char *zDoc,
  int nDoc,
  int(*x)(void*, int, int, const char*, int, int, int)
){
  sqlite4_env *pEnv = (sqlite4_env *)p;
  char *aBuf;
  int nBuf;

  int enc,
  void *p,
  void (*xFunc)(sqlite4_context*,int,sqlite4_value **),
  void (*xDestroy)(void *)
);

/*
** CAPIREF: Matchinfo APIs.
**
** Special functions that may be called from within matchinfo UDFs. All
** return an SQLite error code - SQLITE4_OK if successful, or some other
** error code otherwise.
**
** sqlite4_mi_column_count():
**   Set *pn to the number of columns in the queried table.
**

**   identified by parameters iC and iS.
**
**   Parameter iP may also be negative. In this case, the output value is
**   set to the total number of occurrences of all query phrases in the
**   current row, subject to the constraints imposed by iC and iS.
**
** sqlite4_mi_match_detail():
**   This function is used to access the details of the iMatch'th match
**   (of any phrase) in the current row. Matches are sorted in order of
**   occurrence. If parameter iMatch is equal to or greater than the number 
**   of matches in the current row, SQLITE_NOTFOUND is returned. Otherwise,
**   unless an error occurs, SQLITE4_OK is returned and the *piOff, *piC, *piS,
**   and *piP output parameters are set to the token offset, column number,
**   stream number and phrase number respectively.
**
**   It is anticipated that this function be used to iterate through matches 
**   in order of occurrence. It is optimized so that it is fastest when 
**   called with the iMatch parameter set to 0, P or P+1, where P is the 
**   iMatch value passed to the previous call.
**
** sqlite4_mi_column_value():
**   Set *ppVal to point to an sqlite4_value object containing the value
**   read from column iCol of the current row. This object is valid until
**   the function callback returns.
*/
int sqlite4_mi_column_count(sqlite4_context *, int *pn);
int sqlite4_mi_phrase_count(sqlite4_context *, int *pn);
int sqlite4_mi_stream_count(sqlite4_context *, int *pn);

int sqlite4_mi_total_size(sqlite4_context *, int iC, int iS, int *pn);
int sqlite4_mi_total_rows(sqlite4_context *, int *pn);

int sqlite4_mi_row_count(sqlite4_context *, int iC, int iS, int iP, int *pn);

int sqlite4_mi_size(sqlite4_context *, int iC, int iS, int *pn);
int sqlite4_mi_match_count(sqlite4_context *, int iC, int iS, int iP, int *pn);
int sqlite4_mi_match_detail(
    sqlite4_context *, int iMatch, int *piOff, int *piC, int *piS, int *piP
);
int sqlite4_mi_column_value(sqlite4_context *, int iC, sqlite4_value **ppVal);

int sqlite4_mi_tokenize(sqlite4_context *, const char *, int, void *,
  int(*x)(void *, int, int, const char *, int, int, int)
);



/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/

/*
** An instance of this structure is used as the p4 argument to some fts5
** related vdbe opcodes.
*/
struct Fts5Info {
  int iDb;                        /* Database containing this index */
  int iRoot;                      /* Root page number of index */
  int iTbl;                       /* Root page number of indexed table */
  int nCol;                       /* Number of columns in indexed table */
  char **azCol;                   /* Column names for table */
  Fts5Tokenizer *pTokenizer;      /* Tokenizer module */
  sqlite4_tokenizer *p;           /* Tokenizer instance */
};

int sqlite4WalkExpr(Walker*, Expr*);