return rc;
}

/*
** Return the number of phrases in expression pExpr.
*/
int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
  return pExpr->nPhrase;
}

/*
** Return the number of terms in the iPhrase'th phrase in pExpr.
*/
int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;

  return rc;
}

/*
** Return the number of phrases in expression pExpr.
*/
int sqlite3Fts5ExprPhraseCount(Fts5Expr *pExpr){
  return (pExpr ? pExpr->nPhrase : 0);
}

/*
** Return the number of terms in the iPhrase'th phrase in pExpr.
*/
int sqlite3Fts5ExprPhraseSize(Fts5Expr *pExpr, int iPhrase){
  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ) return 0;

      int nSuffix;                /* Size of term suffix */

      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
      nTerm = strlen(zTerm);

      /* Decide if the term will fit on the current leaf. If it will not, 
      ** flush the leaf to disk here.  */
      if( (pBuf->n + nTerm + 2) > pgsz ){
        fts5WriteFlushLeaf(p, &writer);
        pBuf = &writer.writer.buf;
        if( (nTerm + 32) > pBuf->nSpace ){
          fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
          if( p->rc ) break;
        }
      }

      int nSuffix;                /* Size of term suffix */

      sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
      nTerm = strlen(zTerm);

      /* Decide if the term will fit on the current leaf. If it will not, 
      ** flush the leaf to disk here.  */
      if( pBuf->n>4 && (pBuf->n + nTerm + 2) > pgsz ){
        fts5WriteFlushLeaf(p, &writer);
        pBuf = &writer.writer.buf;
        if( (nTerm + 32) > pBuf->nSpace ){
          fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n);
          if( p->rc ) break;
        }
      }

**
**   If the cursor iterates in descending order of rowid, iFirstRowid
**   is the upper limit (i.e. the "first" rowid visited) and iLastRowid
**   the lower.
*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */





  int ePlan;                      /* FTS5_PLAN_XXX value */
  int bDesc;                      /* True for "ORDER BY rowid DESC" queries */
  i64 iFirstRowid;                /* Return no rowids earlier than this */
  i64 iLastRowid;                 /* Return no rowids later than this */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
  int csrflags;                   /* Mask of cursor flags (see below) */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
  i64 iSpecial;                   /* Result of special query */

  /* "rank" function. Populated on demand from vtab.xColumn(). */
  char *zRank;                    /* Custom rank function */
  char *zRankArgs;                /* Custom rank function args */
  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */
  int nRankArg;                   /* Number of trailing arguments for rank() */
  sqlite3_value **apRankArg;      /* Array of trailing arguments */
  sqlite3_stmt *pRankArgStmt;     /* Origin of objects in apRankArg[] */

  /* Variables used by auxiliary functions */
  i64 iCsrId;                     /* Cursor id */
  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */
  int *aColumnSize;               /* Values for xColumnSize() */

  /* Cache used by auxiliary functions xInst() and xInstCount() */
  int nInstCount;                 /* Number of phrase instances */
  int *aInst;                     /* 3 integers per phrase instance */
};

/*
................................................................................
){
  return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}

/*
** The different query plans.
*/
#define FTS5_PLAN_MATCH          0       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SOURCE         1       /* A source cursor for SORTED_MATCH */
#define FTS5_PLAN_SPECIAL        2       /* An internal query */
#define FTS5_PLAN_SORTED_MATCH   3       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_SCAN           4       /* No usable constraint */
#define FTS5_PLAN_ROWID          5       /* (rowid = ?) */

/*
** Implementation of the xBestIndex method for FTS5 tables. Within the 
** WHERE constraint, it searches for the following:
**
**   1. A MATCH constraint against the special column.
**   2. A MATCH constraint against the "rank" column.
................................................................................
static void fts5CsrNewrow(Fts5Cursor *pCsr){
  CsrFlagSet(pCsr, 
      FTS5CSR_REQUIRE_CONTENT 
    | FTS5CSR_REQUIRE_DOCSIZE 
    | FTS5CSR_REQUIRE_INST 
  );
}







































/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  if( pCursor ){
    Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
    Fts5Cursor **pp;
    Fts5Auxdata *pData;
    Fts5Auxdata *pNext;

    sqlite3_free(pCsr->aInst);
    if( pCsr->pStmt ){
      int eStmt = fts5StmtType(pCsr);
      sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
    }
    if( pCsr->pSorter ){
      Fts5Sorter *pSorter = pCsr->pSorter;
      sqlite3_finalize(pSorter->pStmt);
      sqlite3_free(pSorter);
    }

    if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
      sqlite3Fts5ExprFree(pCsr->pExpr);
    }

    for(pData=pCsr->pAuxdata; pData; pData=pNext){
      pNext = pData->pNext;
      if( pData->xDelete ) pData->xDelete(pData->pPtr);
      sqlite3_free(pData);
    }


    /* Remove the cursor from the Fts5Global.pCsr list */
    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
    *pp = pCsr->pNext;

    sqlite3_finalize(pCsr->pRankArgStmt);
    sqlite3_free(pCsr->apRankArg);

    if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
      sqlite3_free(pCsr->zRank);
      sqlite3_free(pCsr->zRankArgs);
    }
    sqlite3_free(pCsr);
  }
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
................................................................................
** even if we reach end-of-file.  The fts5EofMethod() will be called
** subsequently to determine whether or not an EOF was hit.
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;

  assert( (pCsr->ePlan<2)==
          (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
  );

  if( pCsr->ePlan<2 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
    rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
      CsrFlagSet(pCsr, FTS5CSR_EOF);
    }
    fts5CsrNewrow(pCsr);
................................................................................
  int bOrderByRank;               /* True if ORDER BY rank */
  sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  char **pzErrmsg = pConfig->pzErrmsg;






  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );
  assert( pCsr->zRank==0 );
  assert( pCsr->zRankArgs==0 );

**
**   If the cursor iterates in descending order of rowid, iFirstRowid
**   is the upper limit (i.e. the "first" rowid visited) and iLastRowid
**   the lower.
*/
struct Fts5Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  Fts5Cursor *pNext;              /* Next cursor in Fts5Cursor.pCsr list */
  int *aColumnSize;               /* Values for xColumnSize() */
  i64 iCsrId;                     /* Cursor id */

  /* Zero from this point onwards on cursor reset */
  int ePlan;                      /* FTS5_PLAN_XXX value */
  int bDesc;                      /* True for "ORDER BY rowid DESC" queries */
  i64 iFirstRowid;                /* Return no rowids earlier than this */
  i64 iLastRowid;                 /* Return no rowids later than this */
  sqlite3_stmt *pStmt;            /* Statement used to read %_content */
  Fts5Expr *pExpr;                /* Expression for MATCH queries */
  Fts5Sorter *pSorter;            /* Sorter for "ORDER BY rank" queries */
  int csrflags;                   /* Mask of cursor flags (see below) */

  i64 iSpecial;                   /* Result of special query */

  /* "rank" function. Populated on demand from vtab.xColumn(). */
  char *zRank;                    /* Custom rank function */
  char *zRankArgs;                /* Custom rank function args */
  Fts5Auxiliary *pRank;           /* Rank callback (or NULL) */
  int nRankArg;                   /* Number of trailing arguments for rank() */
  sqlite3_value **apRankArg;      /* Array of trailing arguments */
  sqlite3_stmt *pRankArgStmt;     /* Origin of objects in apRankArg[] */

  /* Auxiliary data storage */

  Fts5Auxiliary *pAux;            /* Currently executing extension function */
  Fts5Auxdata *pAuxdata;          /* First in linked list of saved aux-data */


  /* Cache used by auxiliary functions xInst() and xInstCount() */
  int nInstCount;                 /* Number of phrase instances */
  int *aInst;                     /* 3 integers per phrase instance */
};

/*
................................................................................
){
  return fts5InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr);
}

/*
** The different query plans.
*/
#define FTS5_PLAN_MATCH          1       /* (<tbl> MATCH ?) */
#define FTS5_PLAN_SOURCE         2       /* A source cursor for SORTED_MATCH */
#define FTS5_PLAN_SPECIAL        3       /* An internal query */
#define FTS5_PLAN_SORTED_MATCH   4       /* (<tbl> MATCH ? ORDER BY rank) */
#define FTS5_PLAN_SCAN           5       /* No usable constraint */
#define FTS5_PLAN_ROWID          6       /* (rowid = ?) */

/*
** Implementation of the xBestIndex method for FTS5 tables. Within the 
** WHERE constraint, it searches for the following:
**
**   1. A MATCH constraint against the special column.
**   2. A MATCH constraint against the "rank" column.
................................................................................
static void fts5CsrNewrow(Fts5Cursor *pCsr){
  CsrFlagSet(pCsr, 
      FTS5CSR_REQUIRE_CONTENT 
    | FTS5CSR_REQUIRE_DOCSIZE 
    | FTS5CSR_REQUIRE_INST 
  );
}

static void fts5FreeCursorComponents(Fts5Cursor *pCsr){
  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
  Fts5Auxdata *pData;
  Fts5Auxdata *pNext;

  sqlite3_free(pCsr->aInst);
  if( pCsr->pStmt ){
    int eStmt = fts5StmtType(pCsr);
    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
  }
  if( pCsr->pSorter ){
    Fts5Sorter *pSorter = pCsr->pSorter;
    sqlite3_finalize(pSorter->pStmt);
    sqlite3_free(pSorter);
  }

  if( pCsr->ePlan!=FTS5_PLAN_SOURCE ){
    sqlite3Fts5ExprFree(pCsr->pExpr);
  }

  for(pData=pCsr->pAuxdata; pData; pData=pNext){
    pNext = pData->pNext;
    if( pData->xDelete ) pData->xDelete(pData->pPtr);
    sqlite3_free(pData);
  }

  sqlite3_finalize(pCsr->pRankArgStmt);
  sqlite3_free(pCsr->apRankArg);

  if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
    sqlite3_free(pCsr->zRank);
    sqlite3_free(pCsr->zRankArgs);
  }

  memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr));
}


/*
** Close the cursor.  For additional information see the documentation
** on the xClose method of the virtual table interface.
*/
static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
  if( pCursor ){
    Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
    Fts5Cursor **pp;
























    fts5FreeCursorComponents(pCsr);
    /* Remove the cursor from the Fts5Global.pCsr list */
    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
    *pp = pCsr->pNext;








    sqlite3_free(pCsr);
  }
  return SQLITE_OK;
}

static int fts5SorterNext(Fts5Cursor *pCsr){
  Fts5Sorter *pSorter = pCsr->pSorter;
................................................................................
** even if we reach end-of-file.  The fts5EofMethod() will be called
** subsequently to determine whether or not an EOF was hit.
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  int rc = SQLITE_OK;

  assert( (pCsr->ePlan<3)==
          (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) 
  );

  if( pCsr->ePlan<3 ){
    int bSkip = 0;
    if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
    rc = sqlite3Fts5ExprNext(pCsr->pExpr, pCsr->iLastRowid);
    if( sqlite3Fts5ExprEof(pCsr->pExpr) ){
      CsrFlagSet(pCsr, FTS5CSR_EOF);
    }
    fts5CsrNewrow(pCsr);
................................................................................
  int bOrderByRank;               /* True if ORDER BY rank */
  sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  char **pzErrmsg = pConfig->pzErrmsg;

  if( pCsr->ePlan ){
    fts5FreeCursorComponents(pCsr);
    memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
  }

  assert( pCsr->pStmt==0 );
  assert( pCsr->pExpr==0 );
  assert( pCsr->csrflags==0 );
  assert( pCsr->pRank==0 );
  assert( pCsr->zRank==0 );
  assert( pCsr->zRankArgs==0 );

**         "a OR (b AND c)"
**
**     In FTS4, if a single row contains instances of tokens "a" and "c", 
**     but not "b", all instances of "c" are considered matches. In FTS5,
**     they are not (as the "b AND c" sub-tree does not match the current
**     row.
**
**  2) ...


**     
** This file exports a single function that may be called to register the
** matchinfo() implementation with a database handle:
**
**   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
*/

................................................................................
  sqlite3_finalize(pStmt);
  return pRet;
}


/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, 0 is 
** returned. Otherwise, if f is a valid matchinfo flag, the value returned
** is the number of 32-bit integers added to the output array if the
** table has nCol columns and the query nPhrase phrases.
*/
static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
  int ret = 0;
  switch( f ){
    case 'p': ret = 1; break;
    case 'c': ret = 1; break;
    case 'x': ret = 3 * nCol * nPhrase; break;
    case 'y': ret = nCol * nPhrase; break;
    case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
    case 'n': ret = 1; break;
................................................................................
        int nToken;
        rc = pApi->xColumnSize(pFts, i, &nToken);
        aOut[i] = (u32)nToken;
      }
      break;
    }

    case 's':


      memset(aOut, 0, sizeof(u32) * p->nCol);


























      break;

  }
  return rc;
}
 
static Fts5MatchinfoCtx *fts5MatchinfoNew(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................

  nCol = pApi->xColumnCount(pFts);
  nPhrase = pApi->xPhraseCount(pFts);

  nInt = 0;
  for(i=0; zArg[i]; i++){
    int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
    if( n==0 ){
      char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
      sqlite3_result_error(pCtx, zErr, -1);
      sqlite3_free(zErr);
      return 0;
    }
    nInt += n;
  }

**         "a OR (b AND c)"
**
**     In FTS4, if a single row contains instances of tokens "a" and "c", 
**     but not "b", all instances of "c" are considered matches. In FTS5,
**     they are not (as the "b AND c" sub-tree does not match the current
**     row.
**
**  2) For the values returned by 'x' that apply to all rows of the table, 
**     NEAR constraints are not considered. But for the number of hits in
**     the current row, they are.
**     
** This file exports a single function that may be called to register the
** matchinfo() implementation with a database handle:
**
**   int sqlite3Fts5TestRegisterMatchinfo(sqlite3 *db);
*/

................................................................................
  sqlite3_finalize(pStmt);
  return pRet;
}


/*
** Argument f should be a flag accepted by matchinfo() (a valid character
** in the string passed as the second argument). If it is not, -1 is 
** returned. Otherwise, if f is a valid matchinfo flag, the value returned
** is the number of 32-bit integers added to the output array if the
** table has nCol columns and the query nPhrase phrases.
*/
static int fts5MatchinfoFlagsize(int nCol, int nPhrase, char f){
  int ret = -1;
  switch( f ){
    case 'p': ret = 1; break;
    case 'c': ret = 1; break;
    case 'x': ret = 3 * nCol * nPhrase; break;
    case 'y': ret = nCol * nPhrase; break;
    case 'b': ret = ((nCol + 31) / 32) * nPhrase; break;
    case 'n': ret = 1; break;
................................................................................
        int nToken;
        rc = pApi->xColumnSize(pFts, i, &nToken);
        aOut[i] = (u32)nToken;
      }
      break;
    }

    case 's': {
      int nInst;

      memset(aOut, 0, sizeof(u32) * p->nCol);

      rc = pApi->xInstCount(pFts, &nInst);
      for(i=0; rc==SQLITE_OK && i<nInst; i++){
        int iPhrase, iOff, iCol = 0;
        int iNextPhrase;
        int iNextOff;
        int nSeq = 1;
        int j;

        rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
        iNextPhrase = iPhrase+1;
        iNextOff = iOff+pApi->xPhraseSize(pFts, 0);
        for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
          int ip, ic, io;
          rc = pApi->xInst(pFts, j, &ip, &ic, &io);
          if( ic!=iCol || io>iNextOff ) break;
          if( ip==iNextPhrase && io==iNextOff ){
            nSeq++;
            iNextPhrase = ip+1;
            iNextOff = io + pApi->xPhraseSize(pFts, ip);
          }
        }

        if( nSeq>aOut[iCol] ) aOut[iCol] = nSeq;
      }

      break;
    }
  }
  return rc;
}
 
static Fts5MatchinfoCtx *fts5MatchinfoNew(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
................................................................................

  nCol = pApi->xColumnCount(pFts);
  nPhrase = pApi->xPhraseCount(pFts);

  nInt = 0;
  for(i=0; zArg[i]; i++){
    int n = fts5MatchinfoFlagsize(nCol, nPhrase, zArg[i]);
    if( n<0 ){
      char *zErr = sqlite3_mprintf("unrecognized matchinfo flag: %c", zArg[i]);
      sqlite3_result_error(pCtx, zErr, -1);
      sqlite3_free(zErr);
      return 0;
    }
    nInt += n;
  }

    set cnt [list]
    for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
    $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
    lappend res $cnt
  }
  set res
}





proc fts5_test_all {cmd} {
  set res [list]
  lappend res columnsize      [fts5_test_columnsize $cmd]
  lappend res columntext      [fts5_test_columntext $cmd]
  lappend res columntotalsize [fts5_test_columntotalsize $cmd]
  lappend res poslist         [fts5_test_poslist $cmd]
................................................................................
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all

    fts5_test_queryphrase

  } {
    sqlite3_fts5_create_function $db $f $f
  }
}

proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"

    set cnt [list]
    for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
    $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
    lappend res $cnt
  }
  set res
}

proc fts5_test_phrasecount {cmd} {
  $cmd xPhraseCount
}

proc fts5_test_all {cmd} {
  set res [list]
  lappend res columnsize      [fts5_test_columnsize $cmd]
  lappend res columntext      [fts5_test_columntext $cmd]
  lappend res columntotalsize [fts5_test_columntotalsize $cmd]
  lappend res poslist         [fts5_test_poslist $cmd]
................................................................................
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all

    fts5_test_queryphrase
    fts5_test_phrasecount
  } {
    sqlite3_fts5_create_function $db $f $f
  }
}

proc fts5_level_segs {tbl} {
  set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"

    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank;
  } $res

  do_execsql_test 8.2.$tn.3 {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank;
  } $res
}
























finish_test

    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY +rank;
  } $res

  do_execsql_test 8.2.$tn.3 {
    SELECT rowid FROM t8 WHERE t8 MATCH $q ORDER BY rank;
  } $res
}

#-------------------------------------------------------------------------
# Test xPhraseCount() for some different queries.
#
do_test 9.1 {
  execsql { CREATE VIRTUAL TABLE t9 USING fts5(x) }
  foreach x {
    "a b c" "d e f"
  } {
    execsql { INSERT INTO t9 VALUES($x) }
  }
} {}

foreach {tn q cnt} {
  1 {a AND b}      2
  2 {a OR b}       2
  3 {a OR b OR c}  3
  4 {NEAR(a b)}    2
} {
  do_execsql_test 9.2.$tn {
    SELECT fts5_test_phrasecount(t9) FROM t9 WHERE t9 MATCH $q LIMIT 1
  } $cnt
}

finish_test

# 2015 August 05
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5matchinfo

# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 { finish_test ; return }

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

sqlite3_fts5_register_matchinfo db

do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(content);
} 

do_execsql_test 1.1 {
  INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
  INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
  INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
  INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
  SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}

# Now create an FTS4 table that does not specify matchinfo=fts3.
#
do_execsql_test 1.2 {
  CREATE VIRTUAL TABLE t2 USING fts5(content);
  INSERT INTO t2 SELECT * FROM t1;
  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}


#--------------------------------------------------------------------------
# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
#
# The first argument - $tn - is a test identifier. This may be either a
# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix
# is set, just the numeric component (i.e. "1.1").
#
# The second argument is the name of an FTSX table. The third is the 
# full text of a WHERE/MATCH expression to query the table for 
# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a
# key-value list (serialized array) with matchinfo() format specifiers
# as keys, and the results of executing the statement:
#
#   SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr
#
# For example:
#
#   CREATE VIRTUAL TABLE t1 USING fts4;
#   INSERT INTO t1 VALUES('abc');
#   INSERT INTO t1 VALUES('def');
#   INSERT INTO t1 VALUES('abc abc');
#
#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
#     n {3 3}
#     p {1 1}
#     c {1 1}
#     x {{1 3 2} {2 3 2}}
#   }
#
# If the $results list contains keys mapped to "-" instead of a matchinfo()
# result, then this command computes the expected results based on other
# mappings to test the matchinfo() function. For example, the command above
# could be changed to:
#
#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
#     n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}
#     pcx -
#   }
#
# And this command would compute the expected results for matchinfo(t1, 'pcx')
# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and 
# matchinfo(t1, 'x') in order to test 'pcx'.
#
proc do_matchinfo_test {tn tbl expr results} {

  foreach {fmt res} $results {
    if {$res == "-"} continue
    set resarray($fmt) $res
  }

  set nRow 0
  foreach {fmt res} [array get resarray] {
    if {[llength $res]>$nRow} { set nRow [llength $res] }
  }

  # Construct expected results for any formats for which the caller 
  # supplied result is "-".
  #
  foreach {fmt res} $results {
    if {$res == "-"} {
      set res [list]
      for {set iRow 0} {$iRow<$nRow} {incr iRow} {
        set rowres [list]
        foreach c [split $fmt ""] {
          set rowres [concat $rowres [lindex $resarray($c) $iRow]]
        }
        lappend res $rowres
      }
      set resarray($fmt) $res
    }
  }

  # Test each matchinfo() request individually.
  #
  foreach {fmt res} [array get resarray] {
    set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"
    do_execsql_test $tn.$fmt $sql [normalize2 $res]
  }

  # Test them all executed together (multiple invocations of matchinfo()).
  #
  set exprlist [list]
  foreach {format res} [array get resarray] {
    lappend exprlist "mit(matchinfo($tbl, '$format'))"
  }
  set allres [list]
  for {set iRow 0} {$iRow<$nRow} {incr iRow} {
    foreach {format res} [array get resarray] {
      lappend allres [lindex $res $iRow]
    }
  }
  set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"
  do_execsql_test $tn.multi $sql [normalize2 $allres]
}
proc normalize2 {list_of_lists} {
  set res [list]
  foreach elem $list_of_lists {
    lappend res [list {*}$elem]
  }
  return $res
}


do_execsql_test 4.1.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
  INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
  INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
}

do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
  s {{3 0} {0 3}}
}

do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
  p {3 3}
  x {
    {1 1 1   0 1 1   1 1 1   0 1 1   1 1 1   0 1 1}
    {0 1 1   1 1 1   0 1 1   1 1 1   0 1 1   1 1 1}
  }
}

do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
  p {3 3}
  c {2 2}
  x {
    {1 1 1   0 1 1   1 1 1   0 1 1   1 1 1   0 1 1}
    {0 1 1   1 1 1   0 1 1   1 1 1   0 1 1   1 1 1}
  }
  n {2 2}
  l {{5 5} {5 5}}
  a {{5 5} {5 5}}

  s {{3 0} {0 3}}

  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
  xpxsscplax -
}

do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
  p {1 1}
  c {2 2}
  x {
    {0 1 1   1 1 1}
    {1 1 1   0 1 1}
  }
  n {2 2}
  l {{5 5} {5 5}}
  a {{5 5} {5 5}}

  s {{0 1} {1 0}}

  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
  sxsxs -
}

do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }
do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
  x { 
    {0 1 1  1 1 1  0 0 0  0 0 0} 
    {1 1 1  0 1 1  0 0 0  0 0 0}
  }
}
do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} {
  x { 
    {0 1 1  1 1 1  0 0 0  0 0 0}
    {1 1 1  0 1 1  0 0 0  0 0 0}
  }
}

do_execsql_test 4.2.0 {
  CREATE VIRTUAL TABLE t5 USING fts5(content);
  INSERT INTO t5 VALUES('a a a a a');
  INSERT INTO t5 VALUES('a b a b a');
  INSERT INTO t5 VALUES('c b c b c');
  INSERT INTO t5 VALUES('x x x x x');
}
do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         { 
  x {{5 8 2   5 8 2} {3 8 2   3 8 2}}
  s {2 1} 
}
do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }

do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";

# It used to be that the second 'a' token would be deferred. That doesn't
# work any longer.
if 0 {
  do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
    x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
    s {2 1} 
  }
}

do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }

do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }

do_execsql_test 4.5.0 {
  CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
  INSERT INTO t6 VALUES('a', 'b', 'c');
}
do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }


#-------------------------------------------------------------------------
# Test the outcome of matchinfo() when used within a query that does not
# use the full-text index (i.e. lookup by rowid or full-table scan).
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t10 USING fts5(content);
  INSERT INTO t10 VALUES('first record');
  INSERT INTO t10 VALUES('second record');
}
do_execsql_test 7.2 {
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10;
} {blob 8 blob 8}
do_execsql_test 7.3 {
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) FROM t10 WHERE rowid=1;
} {blob 8}
do_execsql_test 7.4 {
  SELECT typeof(matchinfo(t10)), length(matchinfo(t10)) 
  FROM t10 WHERE t10 MATCH 'record'
} {blob 20 blob 20}

#-------------------------------------------------------------------------
# Test a special case - matchinfo('nxa') with many zero length documents. 
# Special because "x" internally uses a statement used by both "n" and "a". 
# This was causing a problem at one point in the obscure case where the
# total number of bytes of data stored in an fts3 table was greater than
# the number of rows. i.e. when the following query returns true:
#
#   SELECT sum(length(content)) < count(*) FROM fts4table;
#
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t11 USING fts5(content);
  INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
  INSERT INTO t11 VALUES('quitealongstringoftext');
  INSERT INTO t11 VALUES('anotherquitealongstringoftext');
  INSERT INTO t11 VALUES('athirdlongstringoftext');
  INSERT INTO t11 VALUES('andonemoreforgoodluck');
}
do_test 8.2 {
  for {set i 0} {$i < 200} {incr i} {
    execsql { INSERT INTO t11 VALUES('') }
  }
  execsql { INSERT INTO t11(t11) VALUES('optimize') }
} {}
do_execsql_test 8.3 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}

#-------------------------------------------------------------------------
breakpoint
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t12 USING fts5(content);
  INSERT INTO t12 VALUES('a b c d');
  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {{0 1 1 0 1 1 1 1 1}}
do_execsql_test 8.2 {
  INSERT INTO t12 VALUES('a d c d');
  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
  {0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
}
do_execsql_test 8.3 {
  INSERT INTO t12 VALUES('a d d a');
  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
  {0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
}

#---------------------------------------------------------------------------
# Test for a memory leak
#
do_execsql_test 10.1 {
  DROP TABLE t10;
  CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
  INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
  SELECT t10.rowid, t10.*
    FROM t10
    JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x
   WHERE t10 MATCH x.idx
     AND matchinfo(t10) not null
   GROUP BY t10.rowid
   ORDER BY 1;
} {1 1 one 2 2 two 3 3 three}
  
#---------------------------------------------------------------------------
# Test the 'y' matchinfo flag
#
set sqlite_fts3_enable_parentheses 1
reset_db
do_execsql_test 11.0 {
  CREATE VIRTUAL TABLE tt USING fts3(x, y);
  INSERT INTO tt VALUES('c d a c d d', 'e a g b d a');   -- 1
  INSERT INTO tt VALUES('c c g a e b', 'c g d g e c');   -- 2
  INSERT INTO tt VALUES('b e f d e g', 'b a c b c g');   -- 3
  INSERT INTO tt VALUES('a c f f g d', 'd b f d e g');   -- 4
  INSERT INTO tt VALUES('g a c f c f', 'd g g b c c');   -- 5
  INSERT INTO tt VALUES('g a c e b b', 'd b f b g g');   -- 6
  INSERT INTO tt VALUES('f d a a f c', 'e e a d c f');   -- 7
  INSERT INTO tt VALUES('a c b b g f', 'a b a e d f');   -- 8
  INSERT INTO tt VALUES('b a f e c c', 'f d b b a b');   -- 9
  INSERT INTO tt VALUES('f d c e a c', 'f a f a a f');   -- 10
}

db func mit mit
foreach {tn expr res} {
  1 "a" {
      1 {1 2}   2 {1 0}   3 {0 1}   4 {1 0}   5 {1 0}
      6 {1 0}   7 {2 1}   8 {1 2}   9 {1 1}  10 {1 3}
  }

  2 "b" {
      1 {0 1}   2 {1 0}   3 {1 2}   4 {0 1}   5 {0 1}
      6 {2 2}             8 {2 1}   9 {1 3}            
  }

  3 "y:a" {
      1 {0 2}             3 {0 1}                    
                7 {0 1}   8 {0 2}   9 {0 1}  10 {0 3}
  }

  4 "x:a" {
      1 {1 0}   2 {1 0}             4 {1 0}   5 {1 0}
      6 {1 0}   7 {2 0}   8 {1 0}   9 {1 0}  10 {1 0}
  }

  5 "a OR b" {
      1 {1 2 0 1}   2 {1 0 1 0}   3 {0 1 1 2}   4 {1 0 0 1}   5 {1 0 0 1}
      6 {1 0 2 2}   7 {2 1 0 0}   8 {1 2 2 1}   9 {1 1 1 3}  10 {1 3 0 0}
  }

  6 "a AND b" {
      1 {1 2 0 1}   2 {1 0 1 0}   3 {0 1 1 2}   4 {1 0 0 1}   5 {1 0 0 1}
      6 {1 0 2 2}                 8 {1 2 2 1}   9 {1 1 1 3}              
  }

  7 "a OR (a AND b)" {
      1 {1 2 1 2 0 1}   2 {1 0 1 0 1 0}   3 {0 1 0 1 1 2}   4 {1 0 1 0 0 1}   
      5 {1 0 1 0 0 1}   6 {1 0 1 0 2 2}   7 {2 1 0 0 0 0}   8 {1 2 1 2 2 1}   
      9 {1 1 1 1 1 3}  10 {1 3 0 0 0 0}
  }

} {
  do_execsql_test 11.1.$tn.1  {
    SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
  } $res

  set r2 [list]
  foreach {rowid L} $res {
    lappend r2 $rowid
    set M [list]
    foreach {a b} $L {
      lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)]
    }
    lappend r2 $M
  }

  do_execsql_test 11.1.$tn.2  {
    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
  } $r2

  do_execsql_test 11.1.$tn.2  {
    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
  } $r2
}
set sqlite_fts3_enable_parentheses 0

#---------------------------------------------------------------------------
# Test the 'b' matchinfo flag
#
set sqlite_fts3_enable_parentheses 1
reset_db
db func mit mit

do_test 12.0 {
  set cols [list]
  for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
  execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"
} {}

do_execsql_test 12.1 {
  INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
  SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
} [list [list [expr 1<<4] [expr 1<<(45-32)]]]

set sqlite_fts3_enable_parentheses 0
finish_test