SQLite

  }

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pCtx, rc);
  }
}

static int fts5TestCallback(
  void *pContext,                 /* Pointer to Fts5Buffer object */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
  int iStart,                     /* Start offset of token */
  int iEnd,                       /* End offset of token */
  int iPos                        /* Position offset of token */
){
  int rc = SQLITE_OK;
  Fts5Buffer *pBuf = (Fts5Buffer*)pContext;
  if( pBuf->n!=0 ){
    sqlite3Fts5BufferAppendString(&rc, pBuf, " ");
  }
  sqlite3Fts5BufferAppendListElem(&rc, pBuf, pToken, nToken);
  return rc;
}


static void fts5TestFunction(
  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
  Fts5Context *pFts,              /* First arg to pass to pApi functions */
  sqlite3_context *pCtx,          /* Context for returning result/error */
  int nVal,                       /* Number of values in apVal[] array */
  sqlite3_value **apVal           /* Array of trailing arguments */
){
  Fts5Buffer s;                   /* Build up text to return here */
  int nCol;                       /* Number of columns in table */
  int nPhrase;                    /* Number of phrases in query */
  i64 iRowid;                     /* Rowid of current row */
  const char *zReq = 0;
  int rc = SQLITE_OK;
  int i;

  if( nVal>=1 ){
    zReq = (const char*)sqlite3_value_text(apVal[0]);
  }

  memset(&s, 0, sizeof(Fts5Buffer));
  nCol = pApi->xColumnCount(pFts);

  /*
  ** xColumnTotalSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "columntotalsize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntotalsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      i64 colsz = 0;
      rc = pApi->xColumnTotalSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  /*
  ** xColumnCount()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columncount ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columncount") ){
    nCol = pApi->xColumnCount(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nCol);
  }

  /*
  ** xColumnSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columnsize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columnsize") ){
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      int colsz = 0;
      rc = pApi->xColumnSize(pFts, i, &colsz);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", i==0?"":" ", colsz);
    }
    if( zReq==0 && nCol>1 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
  }

  /*
  ** xColumnText()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " columntext ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "columntext") ){
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " ");
      sqlite3Fts5BufferAppendListElem(&rc, &s, z, n);
    }
  }

  /*
  ** xInst()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " inst ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "inst") ){
    int nInst;
    rc = pApi->xInstCount(pFts, &nInst);
    for(i=0; rc==SQLITE_OK && i<nInst; i++){
      int iPhrase, iCol, iOff;
      rc = pApi->xInst(pFts, i, &iPhrase, &iCol, &iOff);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d.%d.%d",
          (i==0 ? "" : " "), iPhrase, iCol, iOff
      );
    }
  }

  /*
  ** xPhraseCount()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasecount ");
  nPhrase = pApi->xPhraseCount(pFts);
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasecount") ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nPhrase);
  }

  /*
  ** xPhraseSize()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " phrasesize ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "phrasesize") ){
    if( nPhrase==1 ){
      int nSize = pApi->xPhraseSize(pFts, 0);
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%d", nSize);
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{");
      for(i=0; i<nPhrase; i++){
        int nSize = pApi->xPhraseSize(pFts, i);
        sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s%d", (i==0?"":" "), nSize);
      }
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "}");
    }
  }

  /*
  ** xPoslist()
  */
  if( zReq==0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s, " poslist ");
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "poslist") ){
    int bParen = 0;
    Fts5Buffer s3;
    memset(&s3, 0, sizeof(s3));

    for(i=0; i<nPhrase; i++){
      Fts5Buffer s2;                  /* List of positions for phrase/column */
      int j = 0;
      i64 iPos = 0;
      int nElem = 0;

      memset(&s2, 0, sizeof(s2));
      while( 0==pApi->xPoslist(pFts, i, &j, &iPos) ){
        int iOff = FTS5_POS2OFFSET(iPos);
        int iCol = FTS5_POS2COLUMN(iPos);
        if( nElem!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &s2, " ");
        sqlite3Fts5BufferAppendPrintf(&rc, &s2, "%d.%d", iCol, iOff);
        nElem++;
      }

      if( i!=0 ){
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, " ");
      }
      if( nElem==1 ){
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "%s", (const char*)s2.p);
      }else{
        sqlite3Fts5BufferAppendPrintf(&rc, &s3, "{%s}", (const char*)s2.p);
        bParen = 1;
      }
      sqlite3_free(s2.p);
    }

    if(zReq==0 && (nPhrase>1 || bParen) ){
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "{%s}", (const char*)s3.p);
    }else{
      sqlite3Fts5BufferAppendPrintf(&rc, &s, "%s", (const char*)s3.p);
    }
    sqlite3_free(s3.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendPrintf(&rc, &s, " queryphrase ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "queryphrase") ){
    int ic, ip;
    int *anVal = 0;
    Fts5Buffer buf1;
    memset(&buf1, 0, sizeof(Fts5Buffer));

    if( rc==SQLITE_OK ){
      anVal = (int*)pApi->xGetAuxdata(pFts, 0);
      if( anVal==0 ){
        rc = fts5GatherTotals(pApi, pFts, &anVal);
        if( rc==SQLITE_OK ){
          rc = pApi->xSetAuxdata(pFts, (void*)anVal, sqlite3_free);
        }
      }
    }

    for(ip=0; rc==SQLITE_OK && ip<nPhrase; ip++){
      if( ip>0 ) sqlite3Fts5BufferAppendString(&rc, &buf1, " ");
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "{");
      for(ic=0; ic<nCol; ic++){
        int iVal = anVal[ip * nCol + ic];
        sqlite3Fts5BufferAppendPrintf(&rc, &buf1, "%s%d", ic==0?"":" ", iVal);
      }
      if( nCol>1 ) sqlite3Fts5BufferAppendString(&rc, &buf1, "}");
    }

    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf1.p, buf1.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf1.p);
    }
    sqlite3_free(buf1.p);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowid ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowid") ){
    iRowid = pApi->xRowid(pFts);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", iRowid);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " rowcount ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "rowcount") ){
    i64 nRow;
    rc = pApi->xRowCount(pFts, &nRow);
    sqlite3Fts5BufferAppendPrintf(&rc, &s, "%lld", nRow);
  }

  if( zReq==0 ){
    sqlite3Fts5BufferAppendString(&rc, &s, " tokenize ");
  }
  if( 0==zReq || 0==sqlite3_stricmp(zReq, "tokenize") ){
    Fts5Buffer buf;
    memset(&buf, 0, sizeof(buf));
    for(i=0; rc==SQLITE_OK && i<nCol; i++){
      const char *z;
      int n;
      rc = pApi->xColumnText(pFts, i, &z, &n);
      if( rc==SQLITE_OK ){
        Fts5Buffer buf1;
        memset(&buf1, 0, sizeof(Fts5Buffer));
        rc = pApi->xTokenize(pFts, z, n, (void*)&buf1, fts5TestCallback);
        if( i!=0 ) sqlite3Fts5BufferAppendPrintf(&rc, &buf, " ");
        sqlite3Fts5BufferAppendListElem(&rc, &buf, (const char*)buf1.p, buf1.n);
        sqlite3_free(buf1.p);
      }
    }
    if( zReq==0 ){
      sqlite3Fts5BufferAppendListElem(&rc, &s, (const char*)buf.p, buf.n);
    }else{
      sqlite3Fts5BufferAppendString(&rc, &s, (const char*)buf.p);
    }
    sqlite3_free(buf.p);
  }

  if( rc==SQLITE_OK ){
    sqlite3_result_text(pCtx, (const char*)s.p, -1, SQLITE_TRANSIENT);
  }else{
    sqlite3_result_error_code(pCtx, rc);
  }
  sqlite3Fts5BufferFree(&s);
}

int sqlite3Fts5AuxInit(fts5_api *pApi){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
    void *pUserData;              /* User-data pointer */
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
    { "bm25debug", (void*)1, fts5Bm25Function,    0 },
    { "snippet",   0, fts5SnippetFunction, 0 },
    { "fts5_test", 0, fts5TestFunction,    0 },
    { "highlight", 0, fts5HighlightFunction, 0 },
    { "bm25",      0, fts5Bm25Function,    0 },
  };

  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */

};

typedef struct F5tApi F5tApi;
struct F5tApi {
  const Fts5ExtensionApi *pApi;
  Fts5Context *pFts;
};




















































/*
**      api sub-command... 
**
** Description...
*/
static int xF5tApi(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  struct Sub {
    const char *zName;
    int nArg;
    const char *zMsg;
  } aSub[] = {
    { "xRowid",      0, "" },





    { "xInstCount",  0, "" },
    { "xInst",       1, "IDX" },

    { "xColumnText", 1, "COL" },
    { "xColumnSize", 1, "COL" },


  };

  int rc;
  int iSub = 0;
  F5tApi *p = (F5tApi*)clientData;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND");
    return TCL_ERROR;
  }

  rc = Tcl_GetIndexFromObjStruct(
      interp, objv[1], aSub, sizeof(aSub[0]), "SUB-COMMAND", 0, &iSub
  );
  if( rc!=TCL_OK ) return rc;
  if( aSub[iSub].nArg!=objc-2 ){
    Tcl_WrongNumArgs(interp, 1, objv, aSub[iSub].zMsg);
    return TCL_ERROR;
  }


  switch( iSub ){








    case 0: { /* xRowid */
      sqlite3_int64 iRowid = p->pApi->xRowid(p->pFts);


      Tcl_SetObjResult(interp, Tcl_NewWideIntObj(iRowid));

      break;
    }










































    case 1: { /* xInstCount */
      int nInst;
      rc = p->pApi->xInstCount(p->pFts, &nInst);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(nInst));
      }
      break;
    }

    case 2: { /* xInst */
      int iIdx, ip, ic, io;
      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xInst(p->pFts, iIdx, &ip, &ic, &io);
      if( rc==SQLITE_OK ){
        Tcl_Obj *pList = Tcl_NewObj();
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ip));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(ic));
        Tcl_ListObjAppendElement(interp, pList, Tcl_NewIntObj(io));
        Tcl_SetObjResult(interp, pList);
      }
      break;
    }





    case 3: { /* xColumnText */
      const char *z = 0;
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnText(p->pFts, iCol, &z, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
      }
      break;
    }

    case 4: { /* xColumnSize */
      int n = 0;
      int iCol;
      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
        return TCL_ERROR;
      }
      rc = p->pApi->xColumnSize(p->pFts, iCol, &n);
      if( rc==SQLITE_OK ){
        Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
      }
      break;
    }















    default: 
      assert( 0 );
      break;
  }


  if( rc!=SQLITE_OK ){
    Tcl_AppendResult(interp, "error in api call", 0);
    return TCL_ERROR;
  }

  return TCL_OK;

  foreach {id x y} $data {
    execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
  }
} {}

# Usage:
#
#   poslist aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc poslist {aCol args} {
  set O(-near) 10
  set O(-col)  -1


  set nOpt [lsearch -exact $args --]
  if {$nOpt<0} { error "no -- option" }

  foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
    if {[info exists O($k)]==0} { error "unrecognized option $k" }
    set O($k) $v
  }







  # Set $phraselist to be a list of phrases. $nPhrase its length.
  set phraselist [lrange $args [expr $nOpt+1] end]
  set nPhrase [llength $phraselist]

  for {set j 0} {$j < [llength $aCol]} {incr j} {
    for {set i 0} {$i < $nPhrase} {incr i} { 

          set A($iCol,$iPhrase) [lsort -integer -uniq $A($iCol,$iPhrase)]
        }
      }
    }
  }

  set res [list]


  for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
    set plist [list]
    for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
      foreach a $A($iCol,$iPhrase) {
        lappend plist "$iCol.$a"
      }
    }
    lappend res $plist

  }

    #puts $res
  return $res
}

# Usage:
#
#   nearset aCol ?-near N? ?-col C? -- phrase1 phrase2...
#
proc nearset {args} {
  set plist [poslist {*}$args]
  return [expr [llength [lindex $plist 0]]>0]
}












# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
# 
#   CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global 

  #puts $tclexpr
  foreach {id x y} $::data {
    set cols [list $x $y]
    if $tclexpr {
      if {$bPos} {
        set N [regexp -all -inline {\[nearset [^\]]*\]} $tclexpr]
        set rowres [list]

        foreach phrase $N {
          set cmd "poslist [string range $phrase 9 end-1]"

          set pos [eval $cmd]
          set rowres [concat $rowres $pos]
        }

        lappend res [list $id $rowres]
      } else {
        lappend res $id
      }
    }
  }

  if {$bAsc} {
    set res [lsort -integer -increasing -index 0 $res]
  } else {
    set res [lsort -integer -decreasing -index 0 $res]
  }

  return [concat {*}$res]
}

#
# End of test code
#-------------------------------------------------------------------------











#-------------------------------------------------------------------------
# Test phrase queries.
#
foreach {tn phrase} {
  1 "o"
  2 "b q"
  3 "e a e"
  4 "m d g q q b k b w f q q p p"
  5 "l o o l v v k"
  6 "a"
  7 "b"
  8 "c"
  9 "no"
  10 "L O O L V V K"
} {
  set expr "\"$phrase\""
  set res [matchdata 1 $expr]

  do_execsql_test 1.2.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Test some AND and OR queries.
#
foreach {tn expr} {

  2.3 "d+c OR u"
  2.4 "d+c OR u+d"

  3.1 { a AND b AND c }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 2.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Queries on a specific column.
#
foreach {tn expr} {
  1 "x:a"
  2 "y:a"
  3 "x:b"
  4 "y:b"
} {
  set res [matchdata 1 $expr]
  do_execsql_test 3.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

#-------------------------------------------------------------------------
# Some NEAR queries.
#
foreach {tn expr} {
  1 "NEAR(a b)"
  2 "NEAR(r c)"
  2 { NEAR(r c, 5) }
  3 { NEAR(r c, 3) }
  4 { NEAR(r c, 2) }
  5 { NEAR(r c, 0) }
  6 { NEAR(a b c) }
  7 { NEAR(a b c, 8) }
  8  { x : NEAR(r c) }
  9  { y : NEAR(r c) }
} {
  set res [matchdata 1 $expr]
  do_execsql_test 4.1.$tn.[llength $res] { 
    SELECT rowid, fts5_test(xx, 'poslist') FROM xx WHERE xx match $expr
  } $res
}

do_test 4.1  { poslist {{a b c}} -- a } {0.0}
do_test 4.2  { poslist {{a b c}} -- c } {0.2}

foreach {tn expr tclexpr} {
  1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
} {
  do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
}

  INSERT INTO t1 VALUES('hello', 'hello');
}

do_execsql_test 1.4.2 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'hello' ORDER BY rowid ASC;
} {1 2 4}



#-------------------------------------------------------------------------
# 
do_execsql_test 2.0 {
  CREATE VIRTUAL TABLE t2 USING fts5(x, y);
  INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
  INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}

do_execsql_test 2.1 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'm' ORDER BY rowid;
} {
  1 {{0.5 1.0 1.2}} 
  2 {{0.7 1.5}}
}

do_execsql_test 2.2 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'u OR q' ORDER BY rowid;
} {
  1 {0.0 {}}
  2 {{} {0.2 0.10}}
}

do_execsql_test 2.3 {
  SELECT rowid, fts5_test(t2, 'poslist') FROM t2 
  WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
  1 {{1.3 1.7}}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE t3 USING fts5(x, y);
  INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
  INSERT INTO t3 VALUES( 'r c', '');
}

do_execsql_test 3.1 {
  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
  1 {{0.6 0.10 0.12} {0.9 0.15}}
}

do_execsql_test 3.2 {
  SELECT rowid, fts5_test(t3, 'poslist') FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
  2 {0.0 0.1}
}

do_execsql_test 3.3 {
  INSERT INTO t3 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
  SELECT rowid, fts5_test(t3, 'poslist') 
  FROM t3 WHERE t3 MATCH 'a OR b AND c';
} {
  3 {0.5 {} {}} 
  1 {{0.6 0.10 0.12} {0.9 0.15} 1.2}
}

#-------------------------------------------------------------------------
# 
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts5(x, y);
  INSERT INTO t4 
  VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}

do_execsql_test 4.1 {
  SELECT rowid, fts5_test(t4, 'poslist') FROM t4 WHERE t4 MATCH 'a OR b AND c';
} {
  1 {0.5 {} {}} 
}

#-------------------------------------------------------------------------
# Test that the xColumnSize() and xColumnAvgsize() APIs work.
#

reset_db


do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t5 USING fts5(x, y);
  INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
  INSERT INTO t5 VALUES('', 'a');
  INSERT INTO t5 VALUES('a', '');
}
do_execsql_test 5.2 {
  SELECT rowid, fts5_test(t5, 'columnsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {1 0}
  2 {0 1}
  1 {4 6}
}

do_execsql_test 5.2 {
  SELECT rowid, fts5_test(t5, 'columntext') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {a {}}
  2 {{} a}
  1 {{a b c d} {e f g h i j}}
}

do_execsql_test 5.3 {
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {5 7}
  2 {5 7}
  1 {5 7}
}

do_execsql_test 5.4 {
  INSERT INTO t5 VALUES('x y z', 'v w x y z');
  SELECT rowid, fts5_test(t5, 'columntotalsize') FROM t5 WHERE t5 MATCH 'a'
  ORDER BY rowid DESC;
} {
  3 {8 12}
  2 {8 12}
  1 {8 12}
}

#-------------------------------------------------------------------------
# Test the xTokenize() API
#
reset_db

do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t6 USING fts5(x, y);
  INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
  INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}

do_execsql_test 6.2 {
  SELECT rowid, fts5_test(t6, 'tokenize') FROM t6 WHERE t6 MATCH 't*'
} {
  2 {{horatio than are} {dreamt of in your philosophy}}
  1 {{there are more} {things in heaven and earth}}
}

#-------------------------------------------------------------------------
# Test the xQueryPhrase() API
#
reset_db

do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t7 USING fts5(x, y);
}
do_test 7.2 {
  foreach {x y} {
    {q i b w s a a e l o} {i b z a l f p t e u}
    {b a z t a l o x d i} {b p a d b f h d w y}

  2 b {{3 4}}
  3 c {{2 1}}
  4 d {{2 2}}
  5 {a AND b} {{4 2} {3 4}}
  6 {a OR b OR c OR d} {{4 2} {3 4} {2 1} {2 2}}
} {
  do_execsql_test 7.3.$tn { 
    SELECT fts5_test(t7, 'queryphrase') FROM t7 WHERE t7 MATCH $q LIMIT 1
  } [list $res]
}

do_execsql_test 7.4 {
  SELECT fts5_test(t7, 'rowcount') FROM t7 WHERE t7 MATCH 'a';
} {5 5 5 5}

#do_execsql_test 7.4 {
#  SELECT rowid, bm25debug(t7) FROM t7 WHERE t7 MATCH 'a';
#} {5 5 5 5}
#

    {l j s g k j k h z l} {p v d a t x d e q u} {r l u z b m g k s j}
    {i e y d u x d i n l} {p f z k m m w p u l} {z l p m r q w n d a}
  } {
    execsql { INSERT INTO t1 VALUES($x, $y, $z) }
  }
  set {} {}
} {}



proc do_fts5ag_test {tn E} {
  set q1 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY rank}
  set q2 {SELECT fts5_test(t1) FROM t1 WHERE t1 MATCH $E ORDER BY bm25(t1)}

  set res [execsql $q1]
  set expected [execsql $q2]
  uplevel [list do_test $tn.1 [list set {} $res] $expected]

  append q1 " DESC"
  append q2 " DESC"

# few test cases that deliberately corrupt database files should rescind 
# this setting by invoking "database_can_be_corrupt"
#
database_never_corrupt

source $testdir/thread_common.tcl
source $testdir/malloc_common.tcl