}

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){
  int rc = SQLITE_OK;
  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
  if( pBuf->n!=0 ){
    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
  }
  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
  return rc;
}


static void fts5TestFunction(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  Fts5Buffer s;                   /* Build up text to return here */
  int nCol;                       /* Number of columns in table */
  int nPhrase;                    /* Number of phrases in query */
  i64 iRowid;                     /* Rowid of current row */
  const char *zReq = 0;
  int rc = SQLITE_OK;
  int i;

  if( nVal>=1 ){
    zReq = (const char*)sqlite3_value_text(apVal[0]);
  }

  memset(&s, 0, sizeof(Fts5Buffer));
  nCol = pApi->xColumnCount(pFts);

  /*
  ** xColumnTotalSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      i64 colsz = 0;
      rc = pApi->xColumnTotalSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  /*
  ** xColumnCount()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columncount ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
    nCol = pApi->xColumnCount(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
  }

  /*
  ** xColumnSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columnsize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      int colsz = 0;
      rc = pApi->xColumnSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  /*
  ** xColumnText()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columntext ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntext") ){
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " ");
      sqlite3Fts5BufferAppendListElem(&rc, &s, z, n);
    }
  }

  /*
  ** xInst()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " inst ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "inst") ){
    int nInst;
    rc = pApi->xInstCount(pFts, &nInst);
    for(i=0; rc==SQLITE_OK && i<nInst; i++){
      int iPhrase, iCol, iOff;
      rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d.%d.%d",
          (i==0 ? "" : " "), iPhrase, iCol, iOff
      );
    }
  }

  /*
  ** xPhraseCount()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
  nPhrase = pApi->xPhraseCount(pFts);
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasecount") ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nPhrase);
  }

  /*
  ** xPhraseSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasesize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasesize") ){
    if( nPhrase==1 ){
      int nSize = pApi->xPhraseSize(pFts, 0);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nSize);
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
      for(i=0; i<nPhrase; i++){
        int nSize = pApi->xPhraseSize(pFts, i);
        sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", (i==0?"":" "), nSize);
      }
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
    }
  }

  /*
  ** xPoslist()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " poslist ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "poslist") ){
    int bParen = 0;
    Fts5Buffer s3;
    memset(&s3, 0, sizeof(s3));

    for(i=0; i<nPhrase; i++){
      Fts5Buffer s2;                  /* List of positions for phrase/column */
      int j = 0;
      i64 iPos = 0;
      int nElem = 0;

      memset(&s2, 0, sizeof(s2));
      while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
        int iOff = FTS5_POS2OFFSET(iPos);
        int iCol = FTS5_POS2COLUMN(iPos);
        if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
        sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
        nElem++;
      }

      if( i!=0 ){
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, " ");
      }
      if( nElem==1 ){
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "%s", (const char*)s2.p);
      }else{
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "{%s}", (const char*)s2.p);
        bParen = 1;
      }
      sqlite3_free(s2.p);
    }

    if(zReq==0 && (nPhrase>1 || bParen) ){
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{%s}", (const char*)s3.p);
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
    }
    sqlite3_free(s3.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){
    int ic, ip;
    int *anVal = 0;
    Fts5Buffer buf1;
    memset(&buf1, 0, sizeof(Fts5Buffer));

    if( rc==SQLITE_OK ){
      anVal = (int*)pApi->xGetAuxdata(pFts, 0);
      if( anVal==0 ){
        rc = fts5GatherTotals(pApi, pFts, &anVal);
        if( rc==SQLITE_OK ){
          rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free);
        }
      }
    }

    for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
      if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " ");
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{");
      for(ic=0; ic<nCol; ic++){
        int iVal = anVal[ip * nCol + ic];
        sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal);
      }
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}");
    }

    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p);
    }
    sqlite3_free(buf1.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowcount ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){
    i64 nRow;
    rc = pApi->xRowCount(pFts, &nRow);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " tokenize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( rc==SQLITE_OK ){
        Fts5Buffer buf1;
        memset(&buf1, 0, sizeof(Fts5Buffer));
        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
        sqlite3_free(buf1.p);
      }
    }
    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
    }
    sqlite3_free(buf.p);
  }

  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3Fts5BufferFree(&s);
}

int sqlite3Fts5AuxInit(fts5_api *pApi){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction,    0 },
    { "highlight", 0, fts5HighlightFunction, 0 },
    { "bm25",      0, fts5Bm25Function,    0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

  }

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}












































































































































































































































































int sqlite3Fts5AuxInit(fts5_api *pApi){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },

    { "highlight", 0, fts5HighlightFunction, 0 },
    { "bm25",      0, fts5Bm25Function,    0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

};

typedef struct F5tApi F5tApi;
struct F5tApi {
  const Fts5ExtensionApi *pApi;
  Fts5Context *pFts;
};




















































/*
**      api sub-command... 
**
** Description...
*/
static int xF5tApi(
................................................................................
  Tcl_Obj *CONST objv[]
){
  struct Sub {
    const char *zName;
    int nArg;
    const char *zMsg;
  } aSub[] = {
    { "xRowid",      0, "" },





    { "xInstCount",  0, "" },
    { "xInst",       1, "IDX" },

    { "xColumnText", 1, "COL" },
    { "xColumnSize", 1, "COL" },


  };

  int rc;
  int iSub = 0;
  F5tApi *p = (F5tApi*)clientData;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");
    return TCL_ERROR;
................................................................................
  );
  if( rc!=TCL_OK ) return rc;
  if( aSub[iSub].nArg!=objc-2 ){
    Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);
    return TCL_ERROR;
  }


  switch( iSub ){
    case 0: { /* xRowid */









      sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);


      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));

      break;
    }










































    case 1: { /* xInstCount */
      int nInst;
      rc = p->pApi->xInstCount(p->pFts, &nInst);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));
      }
      break;
    }

    case 2: { /* xInst */
      int iIdx, ip, ic, io;
      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);
      if( rc==SQLITE_OK ){
        Tcl_Obj *pList = Tcl_NewObj();
................................................................................
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));
        Tcl_SetObjResult(interp, pList);
      }
      break;
    }





    case 3: { /* xColumnText */
      const char *z = 0;
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
      }
      break;
    }

    case 4: { /* xColumnSize */
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnSize(p->pFts, iCol, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
      }
      break;
    }















    default: 
      assert( 0 );
      break;
  }


  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in api call", 0);
    return TCL_ERROR;
  }

  return TCL_OK;

};

typedef struct F5tApi F5tApi;
struct F5tApi {
  const Fts5ExtensionApi *pApi;
  Fts5Context *pFts;
};

static int xTokenizeCb(
  void *pCtx, 
  const char *zToken, int nToken, 
  int iStart, int iEnd, int iPos
){
  F5tFunction *p = (F5tFunction*)pCtx;
  Tcl_Obj *pEval = Tcl_DuplicateObj(p->pScript);
  int rc;

  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iPos));

  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);

  return rc;
}

static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(
  const Fts5ExtensionApi *pApi, 
  Fts5Context *pFts, 
  void *pCtx
){
  F5tFunction *p = (F5tFunction*)pCtx;
  static sqlite3_int64 iCmd = 0;
  Tcl_Obj *pEval;
  int rc;

  char zCmd[64];
  F5tApi sApi;

  sApi.pApi = pApi;
  sApi.pFts = pFts;
  sprintf(zCmd, "f5t_2_%lld", iCmd++);
  Tcl_CreateObjCommand(p->interp, zCmd, xF5tApi, &sApi, 0);

  pEval = Tcl_DuplicateObj(p->pScript);
  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zCmd, -1));
  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);
  Tcl_DeleteCommand(p->interp, zCmd);

  return rc;
}

/*
**      api sub-command... 
**
** Description...
*/
static int xF5tApi(
................................................................................
  Tcl_Obj *CONST objv[]
){
  struct Sub {
    const char *zName;
    int nArg;
    const char *zMsg;
  } aSub[] = {
    { "xColumnCount",      0, "" },
    { "xRowCount",         0, "" },
    { "xColumnTotalSize",  1, "COL" },
    { "xTokenize",         2, "TEXT SCRIPT" },
    { "xPhraseCount",      0, "" },
    { "xPhraseSize",       1, "PHRASE" },
    { "xInstCount",        0, "" },
    { "xInst",             1, "IDX" },
    { "xRowid",            0, "" },
    { "xColumnText",       1, "COL" },
    { "xColumnSize",       1, "COL" },
    { "xQueryPhrase",      2, "PHRASE SCRIPT" },
    { 0, 0, 0}
  };

  int rc;
  int iSub = 0;
  F5tApi *p = (F5tApi*)clientData;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");
    return TCL_ERROR;
................................................................................
  );
  if( rc!=TCL_OK ) return rc;
  if( aSub[iSub].nArg!=objc-2 ){
    Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);
    return TCL_ERROR;
  }

#define CASE(i,str) case i: assert( strcmp(aSub[i].zName, str)==0 );
  switch( iSub ){

    CASE(0, "xColumnCount") {
      int nCol;
      nCol = p->pApi->xColumnCount(p->pFts);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(nCol));
      }
      break;
    }
    CASE(1, "xRowCount") {
      sqlite3_int64 nRow;
      rc = p->pApi->xRowCount(p->pFts, &nRow);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nRow));
      }
      break;
    }
    CASE(2, "xColumnTotalSize") {
      int iCol;
      sqlite3_int64 nSize;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ) return TCL_ERROR;
      rc = p->pApi->xColumnTotalSize(p->pFts, iCol, &nSize);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize));
      }
      break;
    }
    CASE(3, "xTokenize") {
      int nText;
      char *zText = Tcl_GetStringFromObj(objv[2], &nText);
      F5tFunction ctx;
      ctx.interp = interp;
      ctx.pScript = objv[3];
      rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
      if( rc==SQLITE_OK ){
        Tcl_ResetResult(interp);
      }
      return rc;
    }
    CASE(4, "xPhraseCount") {
      int nPhrase;
      nPhrase = p->pApi->xPhraseCount(p->pFts);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(nPhrase));
      }
      break;
    }
    CASE(5, "xPhraseSize") {
      int iPhrase;
      int sz;
      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
        return TCL_ERROR;
      }
      sz = p->pApi->xPhraseSize(p->pFts, iPhrase);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(sz));
      }
      break;
    }
    CASE(6, "xInstCount") {
      int nInst;
      rc = p->pApi->xInstCount(p->pFts, &nInst);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));
      }
      break;
    }
    CASE(7, "xInst") {

      int iIdx, ip, ic, io;
      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);
      if( rc==SQLITE_OK ){
        Tcl_Obj *pList = Tcl_NewObj();
................................................................................
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));
        Tcl_SetObjResult(interp, pList);
      }
      break;
    }
    CASE(8, "xRowid") {
      sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);
      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));
      break;
    }
    CASE(9, "xColumnText") {
      const char *z = 0;
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
      }
      break;
    }

    CASE(10, "xColumnSize") {
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnSize(p->pFts, iCol, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
      }
      break;
    }
    CASE(11, "xQueryPhrase") {
      int iPhrase;
      F5tFunction ctx;
      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ){
        return TCL_ERROR;
      }
      ctx.interp = interp;
      ctx.pScript = objv[3];
      rc = p->pApi->xQueryPhrase(p->pFts, iPhrase, &ctx, xQueryPhraseCb);
      if( rc==SQLITE_OK ){
        Tcl_ResetResult(interp);
      }
      break;
    }

    default: 
      assert( 0 );
      break;
  }
#undef CASE

  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in api call", 0);
    return TCL_ERROR;
  }

  return TCL_OK;

# 2014 Dec 19
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#




proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res
}

proc fts5_test_columnsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    lappend res [$cmd xColumnSize $i]
  }
  set res
}

proc fts5_test_columntext {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    lappend res [$cmd xColumnText $i]
  }
  set res
}

proc fts5_test_columntotalsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    lappend res [$cmd xColumnTotalSize $i]
  }
  set res
}

proc test_append_token {varname token iStart iEnd iPos} {
  upvar $varname var
  lappend var $token
}
proc fts5_test_tokenize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    set tokens [list]
    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
    lappend res $tokens
  }
  set res
}

proc fts5_test_rowcount {cmd} {
  $cmd xRowCount
}

proc test_queryphrase_cb {cnt cmd} {
  upvar $cnt L 
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    foreach {ip ic io} [$cmd xInst $i] break
    set A($ic) 1
  }
  foreach ic [array names A] {
    lset L $ic [expr {[lindex $L $ic] + 1}]
  }
}
proc fts5_test_queryphrase {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    set cnt [list]
    for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
    $cmd xQueryPhrase $i [list test_queryphrase_cb cnt]
    lappend res $cnt
  }
  set res
}

proc fts5_test_all {cmd} {
  set res [list]
  lappend res columnsize      [fts5_test_columnsize $cmd]
  lappend res columntext      [fts5_test_columntext $cmd]
  lappend res columntotalsize [fts5_test_columntotalsize $cmd]
  lappend res poslist         [fts5_test_poslist $cmd]
  lappend res tokenize        [fts5_test_tokenize $cmd]
  lappend res rowcount        [fts5_test_rowcount $cmd]
  set res
}

proc fts5_aux_test_functions {db} {
  foreach f {
    fts5_test_columnsize
    fts5_test_columntext
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_all

    fts5_test_queryphrase
  } {
    sqlite3_fts5_create_function $db $f $f
  }
}

  foreach {id x y} $data {
    execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
  }
} {}

# Usage:
#
#   poslist aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc poslist {aCol args} {
  set O(-near) 10
  set O(-col)  -1


  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
  }







  # Set $phraselist to be a list of phrases. $nPhrase its length.
  set phraselist [lrange $args [expr $nOpt+1] end]
  set nPhrase [llength $phraselist]

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 
................................................................................
          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
        }
      }
    }
  }

  set res [list]


  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    set plist [list]
    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
      foreach a $A($iCol,$iPhrase) {
        lappend plist "$iCol.$a"

      }
    }
    lappend res $plist

  }

    #puts $res
  return $res
}

# Usage:
................................................................................
#
#   nearset aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc nearset {args} {
  set plist [poslist {*}$args]
  return [expr [llength [lindex $plist 0]]>0]
}












# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 
................................................................................
  #puts $tclexpr
  foreach {id x y} $::data {
    set cols [list $x $y]
    if $tclexpr {
      if {$bPos} {
        set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
        set rowres [list]

        foreach phrase $N {
          set cmd "poslist [string range $phrase 9 end-1]"

          set pos [eval $cmd]
          set rowres [concat $rowres $pos]
        }

        lappend res [list $id $rowres]
      } else {
        lappend res $id
      }
    }
  }

................................................................................
  return [concat {*}$res]
}

#
# End of test code
#-------------------------------------------------------------------------











#-------------------------------------------------------------------------
# Test phrase queries.
#
foreach {tn phrase} {
  1 "o"
  2 "b q"
  3 "e a e"
................................................................................
  9 "no"
  10 "L O O L V V K"
} {
  set expr "\"$phrase\""
  set res [matchdata 1 $expr]

  do_execsql_test 1.2.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Test some AND and OR queries.
#
foreach {tn expr} {
................................................................................
  2.3 "d+c OR u"
  2.4 "d+c OR u+d"

  3.1 { a AND b AND c }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 2.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Queries on a specific column.
#
foreach {tn expr} {
................................................................................
  1 "x:a"
  2 "y:a"
  3 "x:b"
  4 "y:b"
} {
  set res [matchdata 1 $expr]
  do_execsql_test 3.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Some NEAR queries.
#
foreach {tn expr} {
................................................................................
  6 { NEAR(a b c) }
  7 { NEAR(a b c, 8) }
  8  { x : NEAR(r c) }
  9  { y : NEAR(r c) }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 4.1.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

do_test 4.1  { poslist {{a b c}} -- a } {0.0}
do_test 4.2  { poslist {{a b c}} -- c } {0.2}

foreach {tn expr tclexpr} {
  1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
} {
  do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
}

  foreach {id x y} $data {
    execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
  }
} {}

# Usage:
#
#   poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
#
proc poslist {aCol args} {
  set O(-near) 10
  set O(-col)  -1
  set O(-pc)   ""

  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
  }

  if {$O(-pc) == ""} {
    set counter 0
  } else {
    upvar $O(-pc) counter
  }

  # Set $phraselist to be a list of phrases. $nPhrase its length.
  set phraselist [lrange $args [expr $nOpt+1] end]
  set nPhrase [llength $phraselist]

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 
................................................................................
          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
        }
      }
    }
  }

  set res [list]
#puts [array names A]

  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {

    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
      foreach a $A($iCol,$iPhrase) {

        lappend res "$counter.$iCol.$a"
      }
    }

    incr counter
  }

    #puts $res
  return $res
}

# Usage:
................................................................................
#
#   nearset aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc nearset {args} {
  set plist [poslist {*}$args]
  return [expr [llength [lindex $plist 0]]>0]
}

proc instcompare {lhs rhs} {
  foreach {p1 c1 o1} [split $lhs .] {}
  foreach {p2 c2 o2} [split $rhs .] {}

  set res [expr $c1 - $c2]
  if {$res==0} { set res [expr $o1 - $o2] }
  if {$res==0} { set res [expr $p1 - $p2] }

  return $res
}

# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 
................................................................................
  #puts $tclexpr
  foreach {id x y} $::data {
    set cols [list $x $y]
    if $tclexpr {
      if {$bPos} {
        set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
        set rowres [list]
        set cnt 0
        foreach phrase $N {
          set arglist [string range $phrase 9 end-1]
          set cmd "poslist [lindex $arglist 0] -pc cnt [lrange $arglist 1 end]"
          set pos [eval $cmd]
          set rowres [concat $rowres $pos]
        }
        set rowres [lsort -command instcompare $rowres]
        lappend res [list $id $rowres]
      } else {
        lappend res $id
      }
    }
  }

................................................................................
  return [concat {*}$res]
}

#
# End of test code
#-------------------------------------------------------------------------

proc fts5_test_poslist {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
    lappend res [string map {{ } .} [$cmd xInst $i]]
  }
  set res
}

sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist

#-------------------------------------------------------------------------
# Test phrase queries.
#
foreach {tn phrase} {
  1 "o"
  2 "b q"
  3 "e a e"
................................................................................
  9 "no"
  10 "L O O L V V K"
} {
  set expr "\"$phrase\""
  set res [matchdata 1 $expr]

  do_execsql_test 1.2.$tn.[llength $res] { 
    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Test some AND and OR queries.
#
foreach {tn expr} {
................................................................................
  2.3 "d+c OR u"
  2.4 "d+c OR u+d"

  3.1 { a AND b AND c }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 2.$tn.[llength $res] { 
    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Queries on a specific column.
#
foreach {tn expr} {
................................................................................
  1 "x:a"
  2 "y:a"
  3 "x:b"
  4 "y:b"
} {
  set res [matchdata 1 $expr]
  do_execsql_test 3.$tn.[llength $res] { 
    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Some NEAR queries.
#
foreach {tn expr} {
................................................................................
  6 { NEAR(a b c) }
  7 { NEAR(a b c, 8) }
  8  { x : NEAR(r c) }
  9  { y : NEAR(r c) }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 4.1.$tn.[llength $res] { 
    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
  } $res
}

do_test 4.1  { poslist {{a b c}} -- a } {0.0.0}
do_test 4.2  { poslist {{a b c}} -- c } {0.0.2}

foreach {tn expr tclexpr} {
  1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
} {
  do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
}

  INSERT INTO t1 VALUES('hello', 'hello');
}

do_execsql_test 1.4.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2 4}



#-------------------------------------------------------------------------
# 
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
  INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
  INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}

do_execsql_test 2.1 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'm' ORDER BY rowid;
} {
  1 {{0.5 1.0 1.2}} 
  2 {{0.7 1.5}}
}

do_execsql_test 2.2 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'u OR q' ORDER BY rowid;
} {
  1 {0.0 {}}
  2 {{} {0.2 0.10}}
}

do_execsql_test 2.3 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
  1 {{1.3 1.7}}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
  INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
  INSERT INTO t3 VALUES( 'r c', '');
}

do_execsql_test 3.1 {
  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
  1 {{0.6 0.10 0.12} {0.9 0.15}}
}

do_execsql_test 3.2 {
  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
  2 {0.0 0.1}
}

do_execsql_test 3.3 {
  INSERT INTO t3 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
  SELECT rowid, fts5_test(t3, 'poslist') 
  FROM t3 WHERE t3 MATCH 'a OR b AND c';
} {
  3 {0.5 {} {}} 
  1 {{0.6 0.10 0.12} {0.9 0.15} 1.2}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
  INSERT INTO t4 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}

do_execsql_test 4.1 {
  SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
  1 {0.5 {} {}} 
}

#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#

reset_db


do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
  INSERT INTO t5 VALUES('', 'a');
  INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
  SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {1 0}
  2 {0 1}
  1 {4 6}
}

do_execsql_test 5.2 {
  SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {5 7}
  2 {5 7}
  1 {5 7}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {8 12}
  2 {8 12}
  1 {8 12}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db

do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
}
do_test 7.2 {
  foreach {x y} {
    {q i b w s a a e l o} {i b z a l f p t e u}
    {b a z t a l o x d i} {b p a d b f h d w y}
................................................................................
  2 b {{3 4}}
  3 c {{2 1}}
  4 d {{2 2}}
  5 {a AND b} {{4 2} {3 4}}
  6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
  do_execsql_test 7.3.$tn { 
    SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1
  } [list $res]
}

do_execsql_test 7.4 {
  SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}

#do_execsql_test 7.4 {
#  SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
#} {5 5 5 5}
#

  INSERT INTO t1 VALUES('hello', 'hello');
}

do_execsql_test 1.4.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2 4}

fts5_aux_test_functions db

#-------------------------------------------------------------------------
# 
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
  INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
  INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}

do_execsql_test 2.1 {
  SELECT rowid, fts5_test_poslist(t2) FROM t2 
  WHERE t2 MATCH 'm' ORDER BY rowid;
} {
  1 {0.0.5 0.1.0 0.1.2} 
  2 {0.0.7 0.1.5}
}

do_execsql_test 2.2 {
  SELECT rowid, fts5_test_poslist(t2) FROM t2 
  WHERE t2 MATCH 'u OR q' ORDER BY rowid;
} {
  1 {0.0.0}
  2 {1.0.2 1.0.10}
}

do_execsql_test 2.3 {
  SELECT rowid, fts5_test_poslist(t2) FROM t2 
  WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
  1 {0.1.3 0.1.7}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
  INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
  INSERT INTO t3 VALUES( 'r c', '');
}

do_execsql_test 3.1 {
  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
}

do_execsql_test 3.2 {
  SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
  2 {0.0.0 1.0.1}
}

do_execsql_test 3.3 {
  INSERT INTO t3 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
  SELECT rowid, fts5_test_poslist(t3) 
  FROM t3 WHERE t3 MATCH 'a OR b AND c';
} {
  3 0.0.5 
  1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15 2.1.2}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
  INSERT INTO t4 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}

do_execsql_test 4.1 {
  SELECT rowid, fts5_test_poslist(t4) FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
  1 0.0.5
}

#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#

reset_db
fts5_aux_test_functions db

do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
  INSERT INTO t5 VALUES('', 'a');
  INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
  SELECT rowid, fts5_test_columnsize(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {1 0}
  2 {0 1}
  1 {4 6}
}

do_execsql_test 5.2 {
  SELECT rowid, fts5_test_columntext(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {5 7}
  2 {5 7}
  1 {5 7}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test_columntotalsize(t5) FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {8 12}
  2 {8 12}
  1 {8 12}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}

do_execsql_test 6.2 {
  SELECT rowid, fts5_test_tokenize(t6) FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db
fts5_aux_test_functions db
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
}
do_test 7.2 {
  foreach {x y} {
    {q i b w s a a e l o} {i b z a l f p t e u}
    {b a z t a l o x d i} {b p a d b f h d w y}
................................................................................
  2 b {{3 4}}
  3 c {{2 1}}
  4 d {{2 2}}
  5 {a AND b} {{4 2} {3 4}}
  6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
  do_execsql_test 7.3.$tn { 
    SELECT fts5_test_queryphrase(t7) FROM t7 WHERE t7 MATCH $q LIMIT 1
  } [list $res]
}

do_execsql_test 7.4 {
  SELECT fts5_test_rowcount(t7) FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}

#do_execsql_test 7.4 {
#  SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
#} {5 5 5 5}
#

    {l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
    {i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
  } {
    execsql { INSERT INTO t1 VALUES($x, $y, $z) }
  }
  set {} {}
} {}



proc do_fts5ag_test {tn E} {
  set q1 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
  set q2 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}

  set res [execsql $q1]
  set expected [execsql $q2]
  uplevel [list do_test $tn.1 [list set {} $res] $expected]

  append q1 " DESC"
  append q2 " DESC"

    {l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
    {i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
  } {
    execsql { INSERT INTO t1 VALUES($x, $y, $z) }
  }
  set {} {}
} {}

fts5_aux_test_functions db

proc do_fts5ag_test {tn E} {
  set q1 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
  set q2 {SELECT fts5_test_all(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}

  set res [execsql $q1]
  set expected [execsql $q2]
  uplevel [list do_test $tn.1 [list set {} $res] $expected]

  append q1 " DESC"
  append q2 " DESC"

# few test cases that deliberately corrupt database files should rescind 
# this setting by invoking "database_can_be_corrupt"
#
database_never_corrupt

source $testdir/thread_common.tcl
source $testdir/malloc_common.tcl

# few test cases that deliberately corrupt database files should rescind 
# this setting by invoking "database_can_be_corrupt"
#
database_never_corrupt

source $testdir/thread_common.tcl
source $testdir/malloc_common.tcl
source $testdir/fts5_common.tcl