SQLite4  Check-in [0e439483d7]

  KVCursor *pCsr;                 /* Cursor used to retrive values */
  Mem *aMem;                      /* Array of column values */

  /* Array of nPhrase*nCol integers. See sqlite4_mi_row_count() for details. */
  int *anRow;
  i64 *aGlobal;

  /* Size of each column of current row (in tokens). */
  int bSzValid;
  int *aSz;
};

/*
** This type is used when reading (decoding) an instance-list.
*/
typedef struct InstanceList InstanceList;
struct InstanceList {

    }else{
      rc = fts5LoadGlobal(pCsr->db, pCsr->pInfo, pCsr->aGlobal);
    }
  }
  return rc;
}

static int fts5CsrLoadSz(Fts5Cursor *pCsr){
  sqlite4 *db = pCsr->db;
  Fts5Info *pInfo = pCsr->pInfo;
  int nVal = pInfo->nCol;
  int rc;
  u8 *aKey;
  int nKey = 0;
  int nPk = pCsr->pExpr->pRoot->nPk;
  KVCursor *pKVCsr = 0;           /* Cursor used to read global record */

  aKey = (u8 *)sqlite4DbMallocZero(db, 10 + nPk);
  if( !aKey ) return SQLITE4_NOMEM;

  nKey = putVarint32(aKey, pInfo->iRoot);
  aKey[nKey++] = 0x00;
  memcpy(&aKey[nKey], pCsr->pExpr->pRoot->aPk, nPk);
  nKey += nPk;

  rc = sqlite4KVStoreOpenCursor(db->aDb[pInfo->iDb].pKV, &pKVCsr);
  if( rc==SQLITE4_OK ){
    rc = sqlite4KVCursorSeek(pKVCsr, aKey, nKey, 0);
    if( rc==SQLITE4_NOTFOUND ){
      rc = SQLITE4_CORRUPT_BKPT;
    }else if( rc==SQLITE4_OK ){
      const u8 *aData = 0;
      int nData = 0;
      rc = sqlite4KVCursorData(pKVCsr, 0, -1, &aData, &nData);
      if( rc==SQLITE4_OK ){
        int i;
        int iOff = 0;
        for(i=0; i<nVal; i++){
          iOff += getVarint32(&aData[iOff], pCsr->aSz[i]);
        }
      }
      pCsr->bSzValid = 1;
    }
    sqlite4KVCursorClose(pKVCsr);
  }

  return rc;
}


/*
** Update an fts index.
*/
int sqlite4Fts5Update(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Description of fts index to update */

      return 0;
    }
  }

  return (p->iCol==pFirst->iCol && p->iOff==iReq);
}

static int fts5StringFindInstances(sqlite4 *db, int iCol, Fts5Str *pStr){

  int i;
  int rc = SQLITE4_OK;
  int bEof = 0;
  int nByte = sizeof(InstanceList) * pStr->nToken;
  InstanceList *aIn;
  InstanceList out;

  if( p1->iCol==p2->iCol && p1->iOff<p2->iOff && (p1->iOff+nNear)>=p2->iOff ){
    return 1;
  }
  return 0;
}

static int fts5StringNearTrim(

  Fts5Str *pTrim,                 /* Trim this instance list */
  Fts5Str *pNext,                 /* According to this one */
  int nNear
){
  if( pNext->nList==0 ){
    pTrim->nList = 0;
  }else{

** is set to true before returning.
**
** If the cursors do not point to a match, then *ppAdvance is set to
** the token of the individual cursor that should be advanced before
** retrying this function.
*/
static int fts5PhraseIsMatch(
  sqlite4 *db,                    /* Database handle */
  Fts5Phrase *pPhrase,            /* Phrase to test */
  int *pbMatch,                   /* OUT: True for a match, false otherwise */
  Fts5Token **ppAdvance           /* OUT: Token to advance before retrying */
){
  const u8 *aPk1 = 0;
  int nPk1 = 0;
  int rc = SQLITE4_OK;

  /* At this point, it is established that all of the token cursors in the
  ** phrase point to an entry with the same primary key. Now figure out if
  ** the various string constraints are met. Along the way, synthesize a 
  ** position list for each Fts5Str object.  */
  for(i=0; rc==SQLITE4_OK && i<pPhrase->nStr; i++){
    Fts5Str *pStr = &pPhrase->aStr[i];
    rc = fts5StringFindInstances(db, pPhrase->iCol, pStr);
  }

  /* Trim the instance lists according to any NEAR constraints.  */
  for(i=1; rc==SQLITE4_OK && i<pPhrase->nStr; i++){
    int n = pPhrase->aiNear[i-1];
    rc = fts5StringNearTrim(&pPhrase->aStr[i], &pPhrase->aStr[i-1], n);
  }
  for(i=pPhrase->nStr-1; rc==SQLITE4_OK && i>0; i--){
    int n = pPhrase->aiNear[i-1];
    rc = fts5StringNearTrim(&pPhrase->aStr[i-1], &pPhrase->aStr[i], n);
  }

  *pbMatch = (pPhrase->aStr[0].nList>0);
  return rc;
}

static int fts5PhraseAdvanceToMatch(sqlite4 *db, Fts5Phrase *pPhrase){
  int rc;
  do {
    int bMatch;
    Fts5Token *pAdvance = 0;
    rc = fts5PhraseIsMatch(db, pPhrase, &bMatch, &pAdvance);
    if( rc!=SQLITE4_OK || bMatch ) break;
    rc = fts5TokenAdvance(db, pAdvance);
  }while( rc==SQLITE4_OK );
  return rc;
}

static int fts5ExprAdvance(sqlite4 *db, Fts5ExprNode *p, int bFirst){
  int rc = SQLITE4_OK;

  switch( p->eType ){
    case TOKEN_PRIMITIVE: {
      Fts5Phrase *pPhrase = p->pPhrase;
      if( bFirst==0 ){
        rc = fts5TokenAdvance(db, &pPhrase->aStr[0].aToken[0]);
      }
      if( rc==SQLITE4_OK ) rc = fts5PhraseAdvanceToMatch(db, pPhrase);
      if( rc==SQLITE4_OK ){
        rc = fts5TokenPk(&pPhrase->aStr[0].aToken[0], &p->aPk, &p->nPk);
      }else{
        p->aPk = 0;
        p->nPk = 0;
        if( rc==SQLITE4_NOTFOUND ) rc = SQLITE4_OK;
      }
      break;
    }

    case TOKEN_AND:
      p->aPk = 0;
      p->nPk = 0;
      rc = fts5ExprAdvance(db, p->pLeft, bFirst);
      if( rc==SQLITE4_OK ) rc = fts5ExprAdvance(db, p->pRight, bFirst);
      while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){
        int res = fts5KeyCompare(
            p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk
        );
        if( res<0 ){
          rc = fts5ExprAdvance(db, p->pLeft, 0);
        }else if( res>0 ){
          rc = fts5ExprAdvance(db, p->pRight, 0);
        }else{
          p->aPk = p->pLeft->aPk;
          p->nPk = p->pLeft->nPk;
          break;
        }
      }
      break;

    case TOKEN_OR: {
      int res = 0;
      if( bFirst==0 ){
        res = fts5KeyCompare(
            p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk
        );
      }
        
      if( res<=0 ) rc = fts5ExprAdvance(db, p->pLeft, bFirst);
      if( rc==SQLITE4_OK && res>=0 ){
        rc = fts5ExprAdvance(db, p->pRight, bFirst);
      }

      res = fts5KeyCompare(
          p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk
      );
      if( res>0 ){
        p->aPk = p->pRight->aPk;

    default: assert( p->eType==TOKEN_NOT );

      p->aPk = 0;
      p->nPk = 0;

      rc = fts5ExprAdvance(db, p->pLeft, bFirst);
      if( bFirst && rc==SQLITE4_OK ){
        rc = fts5ExprAdvance(db, p->pRight, bFirst);
      }

      while( rc==SQLITE4_OK && p->pLeft->aPk && p->pRight->aPk ){
        int res = fts5KeyCompare(
            p->pLeft->aPk, p->pLeft->nPk, p->pRight->aPk, p->pRight->nPk
        );
        if( res<0 ){
          break;
        }else if( res>0 ){
          rc = fts5ExprAdvance(db, p->pRight, 0);
        }else{
          rc = fts5ExprAdvance(db, p->pLeft, 0);
        }
      }

      p->aPk = p->pLeft->aPk;
      p->nPk = p->pLeft->nPk;
      break;
  }

  assert( rc!=SQLITE4_NOTFOUND );
  return rc;
}

int sqlite4Fts5Next(Fts5Cursor *pCsr){
  pCsr->bSzValid = 0;
  return fts5ExprAdvance(pCsr->db, pCsr->pExpr->pRoot, 0);
}

int sqlite4Fts5Open(
  sqlite4 *db,                    /* Database handle */
  Fts5Info *pInfo,                /* Index description */
  const char *zMatch,             /* Match expression */
  int bDesc,                      /* True to iterate in desc. order of PK */
  Fts5Cursor **ppCsr,             /* OUT: New FTS cursor object */
  char **pzErr                    /* OUT: Error message */
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr;
  int nMatch = sqlite4Strlen30(zMatch);

  pCsr = sqlite4DbMallocZero(db, sizeof(Fts5Cursor) + nMatch + 1);

  if( !pCsr ){
    rc = SQLITE4_NOMEM;
  }else{
    pCsr->zExpr = (char *)&pCsr[1];
    memcpy(pCsr->zExpr, zMatch, nMatch);
    pCsr->pInfo = pInfo;
    pCsr->db = db;
    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->iRoot, pInfo->azCol, pInfo->nCol, zMatch, &pCsr->pExpr, pzErr
    );
  }

  if( rc==SQLITE4_OK ){
    /* Open a KV cursor for each term in the expression. Set each cursor
    ** to point to the first entry in the range it will scan.  */
    rc = fts5OpenCursors(db, pInfo, pCsr);
  }
  if( rc!=SQLITE4_OK ){
    sqlite4Fts5Close(db, pCsr);
    pCsr = 0;
  }else{
    rc = fts5ExprAdvance(db, pCsr->pExpr->pRoot, 1);
  }
  *ppCsr = pCsr;
  return rc;
}

/*
** Return true if the cursor passed as the second argument currently points

    rc = SQLITE4_MISUSE;
  }
  return rc;
}

int sqlite4_mi_column_size(sqlite4_context *pCtx, int iCol, int *pnToken){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;

  if( pCsr==0 ){
    rc = SQLITE4_MISUSE;
  }else if( iCol>=pCsr->pInfo->nCol ){
    rc = SQLITE4_ERROR;
  }else{
    if( pCsr->aSz==0 ){
      pCsr->aSz = (int *)sqlite4DbMallocZero(
          pCsr->db, sizeof(int)*pCsr->pInfo->nCol
      );
      if( pCsr->aSz==0 ) rc = SQLITE4_NOMEM;
    }
    if( rc==SQLITE4_OK && pCsr->bSzValid==0 ){
      rc = fts5CsrLoadSz(pCsr);
    }
    if( rc==SQLITE4_OK ){
      assert( pCsr->bSzValid );
      if( iCol>=0 ){
        *pnToken = pCsr->aSz[iCol];
      }else{
        int i;
        int nToken = 0;
        for(i=0; i<pCsr->pInfo->nCol; i++){
          nToken += pCsr->aSz[i];
        }
        *pnToken = nToken;
      }
    }
  }
  return rc;
}

int sqlite4_mi_column_value(
  sqlite4_context *pCtx, 
  int iCol, 

  if( pCtx->pFts ){
    *pnPhrase = pCtx->pFts->pExpr->nPhrase;
  }else{
    rc = SQLITE4_MISUSE;
  }
  return rc;
}

static Fts5Str *fts5FindStr(Fts5ExprNode *p, int *piStr){
  Fts5Str *pRet = 0;
  if( p->eType==TOKEN_PRIMITIVE ){
    int iStr = *piStr;
    if( iStr<p->pPhrase->nStr ){
      pRet = &p->pPhrase->aStr[iStr];
    }else{
      *piStr = iStr - p->pPhrase->nStr;
    }
  }else{
    pRet = fts5FindStr(p->pLeft, piStr);
    if( pRet==0 ) pRet = fts5FindStr(p->pRight, piStr);
  }
  return pRet;
}

int sqlite4_mi_match_count(
  sqlite4_context *pCtx, 
  int iCol,
  int iPhrase,
  int *pnMatch
){
  int rc = SQLITE4_OK;
  Fts5Cursor *pCsr = pCtx->pFts;
  if( pCsr ){
    int nMatch = 0;
    Fts5Str *pStr;
    int iCopy = iCol;
    InstanceList sList;

    pStr = fts5FindStr(pCsr->pExpr->pRoot, &iCopy);
    assert( pStr );

    fts5InstanceListInit(pStr->aList, pStr->nList, &sList);
    while( 0==fts5InstanceListNext(&sList) ){
      if( iCol<0 || sList.iCol==iCol ) nMatch++;
    }
    *pnMatch = nMatch;
  }else{
    rc = SQLITE4_MISUSE;
  }
  return rc;
}

int sqlite4_mi_match_offset(
  sqlite4_context *pCtx, 
  int iCol, 
  int iPhrase, 
  int iMatch, 
  int *piOff
){
  return SQLITE4_OK;
}

int sqlite4_mi_total_match_count(
  sqlite4_context *pCtx,
  int iCol,
  int iPhrase,
  int *pnMatch,
  int *pnDoc,
  int *pnRelevant
){
  return SQLITE4_OK;
}

int sqlite4_mi_total_size(sqlite4_context *pCtx, int iCol, int *pnToken){
  int rc = SQLITE4_OK;
  if( pCtx->pFts ){
    Fts5Cursor *pCsr = pCtx->pFts;
    int nCol = pCsr->pInfo->nCol;

      }
    }
  }else{
    rc = SQLITE4_MISUSE;
  }
  return rc;
}

static void fts5StrLoadRowcounts(Fts5Str *pStr, int *anRow){
  InstanceList sList;

  fts5InstanceListInit(pStr->aList, pStr->nList, &sList);
  while( 0==fts5InstanceListNext(&sList) ){
    anRow[sList.iCol]++;
  }
}


static int fts5ExprLoadRowcounts(
  sqlite4 *db, 
  Fts5Info *pInfo,
  Fts5ExprNode *pNode, 
  int **panRow
){
  int rc = SQLITE4_OK;

  if( pNode ){
    if( pNode->eType==TOKEN_PRIMITIVE ){
      int *anRow = *panRow;
      Fts5Phrase *pPhrase = pNode->pPhrase;

      rc = fts5ExprAdvance(db, pNode, 1);
      while( rc==SQLITE4_OK ){
        int i;
        for(i=0; i<pPhrase->nStr; i++){
          fts5StrLoadRowcounts(&pPhrase->aStr[i], &anRow[i*pInfo->nCol]);
        }
        rc = fts5ExprAdvance(db, pNode, 0);
      }

      *panRow = &anRow[pInfo->nCol * pPhrase->nStr];
    }

    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pInfo, pNode->pLeft, panRow);
    }
    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pInfo, pNode->pLeft, panRow);
    }
  }

  return rc;
}

static int fts5CsrLoadRowcounts(Fts5Cursor *pCsr){
  int rc = SQLITE4_OK;

  if( pCsr->anRow==0 ){
    sqlite4 *db = pCsr->db;
    Fts5Expr *pCopy;
    Fts5Expr *pExpr = pCsr->pExpr;
    Fts5Info *pInfo = pCsr->pInfo;
    int *anRow;

    pCsr->anRow = anRow = (int *)sqlite4DbMallocZero(db, 
        pExpr->nPhrase * pInfo->nCol * sizeof(int)
    );
    if( !anRow ) return SQLITE4_NOMEM;

    rc = fts5ParseExpression(db, pInfo->pTokenizer, pInfo->p, 
        pInfo->iRoot, pInfo->azCol, pInfo->nCol, pCsr->zExpr, &pCopy, 0
    );
    if( rc==SQLITE4_OK ){
      rc = fts5OpenExprCursors(db, pInfo, pExpr->pRoot);
    }

    if( rc==SQLITE4_OK ){
      rc = fts5ExprLoadRowcounts(db, pInfo, pCopy->pRoot, &anRow);
    }

    fts5ExpressionFree(db, pCopy);
  }

  return rc;
}

int sqlite4_mi_row_count(
  sqlite4_context *pCtx,          /* Context object passed to mi function */
  int iCol,                       /* Specific column (or -1) */
  int iPhrase,                    /* Specific phrase (or -1) */
  int *pnRow                      /* Total number of rows */

**
**   http://xapian.org/docs/bm25.html
**
**   http://en.wikipedia.org/wiki/Okapi_BM25
*/

#include "sqliteInt.h"
#include <math.h>                 /* temporary: For log() */

static char fts5Tolower(char c){
  if( c>='A' && c<='Z' ) c = c + ('a' - 'A');
  return c;
}

static int fts5SimpleCreate(

**   in column iCol, or in any column if iCol is negative.
**
**   If parameter iPhrase is equal to or greater than the number of phrases
**   in the current query, or if iCol is equal to or greater than the number
**   of columns in the indexed table, SQLITE4_MISUSE is returned. The value
**   of the output parameter is undefined in this case.
*/

int sqlite4_mi_column_count(sqlite4_context *, int *pnCol);
int sqlite4_mi_phrase_count(sqlite4_context *, int *pnPhrase);

int sqlite4_mi_column_size(sqlite4_context *, int iCol, int *pnToken);
int sqlite4_mi_match_count(sqlite4_context *, int iCol, int iPhrase, int *pn);
int sqlite4_mi_total_size(sqlite4_context *, int iCol, int *pnToken);
int sqlite4_mi_row_count(sqlite4_context *, int iCol, int iPhrase, int *pnRow);

int sqlite4_mi_column_value(sqlite4_context *, int iCol, sqlite4_value **ppVal);
int sqlite4_mi_match_detail(sqlite4_context *, 
    int iCol, int iPhrase, int iMatch, int *piOff, int *piWeight
);




/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/
#ifdef SQLITE4_OMIT_FLOATING_POINT
# undef double