SQLite

*************************************************************************/

/*************************************************************************
** CUSTOM TOKENIZERS
**
** Applications may also register custom tokenizer types. A tokenizer 
** is registered by providing fts5 with a populated instance of the 
** following structure. All structure methods must be defined, setting
** any member of the fts5_tokenizer struct to NULL leads to undefined
** behaviour. The structure methods are expected to function as follows:

**
** xCreate:
**   This function is used to allocate and inititalize a tokenizer instance.
**   A tokenizer instance is required to actually tokenize text.
**
**   The first argument passed to this function is a copy of the (void*)
**   pointer provided by the application when the fts5_tokenizer object

  int *pi,                        /* IN/OUT: Offset within a[] */
  i64 *piOff                      /* IN/OUT: Current offset */
);

/* Malloc utility */
void *sqlite3Fts5MallocZero(int *pRc, int nByte);
char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);

/* Character set tests (like isspace(), isalpha() etc.) */
int sqlite3Fts5IsBareword(char t);

/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/

/**************************************************************************
** Interface to code in fts5_index.c. fts5_index.c contains contains code

      zRet[nIn] = '\0';
    }else{
      *pRc = SQLITE_NOMEM;
    }
  }
  return zRet;
}


/*
** Return true if character 't' may be part of an FTS5 bareword, or false
** otherwise. Characters that may be part of barewords:
**
**   * All non-ASCII characters,
**   * The 52 upper and lower case ASCII characters, and
**   * The 10 integer ASCII characters.
**   * The underscore character "_" (0x5F).
*/
int sqlite3Fts5IsBareword(char t){
  u8 aBareword[128] = {
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x00 .. 0x0F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x10 .. 0x1F */
    0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0,   /* 0x20 .. 0x2F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 0, 0, 0, 0, 0, 0,   /* 0x30 .. 0x3F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x40 .. 0x4F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 1,   /* 0x50 .. 0x5F */
    0, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1,   /* 0x60 .. 0x6F */
    1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 0, 0, 0, 0, 0    /* 0x70 .. 0x7F */
  };

  return (t & 0x80) || aBareword[(int)t];
}

#endif /* SQLITE_ENABLE_FTS5 */

/*
** Argument pIn points to a character that is part of a nul-terminated 
** string. Return a pointer to the first character following *pIn in 
** the string that is not a "bareword" character.
*/
static const char *fts5ConfigSkipBareword(const char *pIn){
  const char *p = pIn;




  while ( sqlite3Fts5IsBareword(*p) ) p++;

  if( p==pIn ) p = 0;
  return p;
}

static int fts5_isdigit(char a){
  return (a>='0' && a<='9');
}



static const char *fts5ConfigSkipLiteral(const char *pIn){
  const char *p = pIn;

  switch( *p ){
    case 'n': case 'N':
      if( sqlite3_strnicmp("null", p, 4)==0 ){
        p = &p[4];
      }else{
        p = 0;
      }
      break;

    case 'x': case 'X':
      p++;
      if( *p=='\'' ){
        p++;
        while( (*p>='a' && *p<='f') 
            || (*p>='A' && *p<='F') 
            || (*p>='0' && *p<='9') 
            ){
          p++;
        }
        if( *p=='\'' && 0==((p-pIn)%2) ){
          p++;
        }else{
          p = 0;
        }
      }else{
        p = 0;
      }
      break;

    case '\'':
      p++;
      while( p ){
        if( *p=='\'' ){
          p++;
          if( *p!='\'' ) break;
        }
        p++;
        if( *p==0 ) p = 0;
      }
      break;

    default:
      /* maybe a number */
      if( *p=='+' || *p=='-' ) p++;
      while( fts5_isdigit(*p) ) p++;

      /* At this point, if the literal was an integer, the parse is 
      ** finished. Or, if it is a floating point value, it may continue
      ** with either a decimal point or an 'E' character. */
      if( *p=='.' && fts5_isdigit(p[1]) ){
        p += 2;
        while( fts5_isdigit(*p) ) p++;
      }
      if( p==pIn ) p = 0;

      break;

  }

  return p;
}

/*
** The first character of the string pointed to by argument z is guaranteed

  int iOut = 0;
  q = z[0];

  /* Set stack variable q to the close-quote character */
  assert( q=='[' || q=='\'' || q=='"' || q=='`' );
  if( q=='[' ) q = ']';  

  while( ALWAYS(z[iIn]) ){
    if( z[iIn]==q ){
      if( z[iIn+1]!=q ){
        /* Character iIn was the close quote. */

        iIn++;
        break;
      }else{
        /* Character iIn and iIn+1 form an escaped quote character. Skip
        ** the input cursor past both and copy a single quote character 
        ** to the output buffer. */
        iIn += 2;
        z[iOut++] = q;
      }
    }else{
      z[iOut++] = z[iIn++];
    }
  }


  z[iOut] = '\0';
  return iIn;
}

/*
** Convert an SQL-style quoted string into a normal string by removing
** the quote characters.  The conversion is done in-place.  If the
** input does not begin with a quote character, then this routine
** is a no-op.

    if( azArg && pSpace ){
      if( pConfig->pTok ){
        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
        rc = SQLITE_ERROR;
      }else{
        for(nArg=0; p && *p; nArg++){
          const char *p2 = fts5ConfigSkipWhitespace(p);
          if( *p2=='\'' ){
            p = fts5ConfigSkipLiteral(p2);
          }else{
            p = fts5ConfigSkipBareword(p2);
          }
          if( p ){
            memcpy(pSpace, p2, p-p2);
            azArg[nArg] = pSpace;

static const char *fts5ConfigGobbleWord(
  int *pRc,                       /* IN/OUT: Error code */
  const char *zIn,                /* Buffer to gobble string/bareword from */
  char **pzOut,                   /* OUT: malloc'd buffer containing str/bw */
  int *pbQuoted                   /* OUT: Set to true if dequoting required */
){
  const char *zRet = 0;




  int nIn = strlen(zIn);
  char *zOut = sqlite3_malloc(nIn+1);

  assert( *pRc==SQLITE_OK );
  *pbQuoted = 0;
  *pzOut = 0;

  if( zOut==0 ){
    *pRc = SQLITE_NOMEM;
  }else{
    memcpy(zOut, zIn, nIn+1);
    if( fts5_isopenquote(zOut[0]) ){
      int ii = fts5Dequote(zOut);
      zRet = &zIn[ii];
      *pbQuoted = 1;
    }else{
      zRet = fts5ConfigSkipBareword(zIn);
      zOut[zRet-zIn] = '\0';
    }
  }

  if( zRet==0 ){
    sqlite3_free(zOut);
  }else{
    *pzOut = zOut;

  }

  return zRet;
}

static int fts5ConfigParseColumn(
  Fts5Config *p, 

/*
** Free the configuration object passed as the only argument.
*/
void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
  if( pConfig ){
    int i;
    if( pConfig->pTok ){
      pConfig->pTokApi->xDelete(pConfig->pTok);
    }
    sqlite3_free(pConfig->zDb);
    sqlite3_free(pConfig->zName);
    for(i=0; i<pConfig->nCol; i++){
      sqlite3_free(pConfig->azCol[i]);
    }

    const char *pArgs; 
    p = fts5ConfigSkipWhitespace(p);
    pArgs = p;
    if( *p!=')' ){
      p = fts5ConfigSkipArgs(p);
      if( p==0 ){
        rc = SQLITE_ERROR;
      }else{
        zRankArgs = sqlite3Fts5MallocZero(&rc, 1 + p - pArgs);
        if( zRankArgs ) memcpy(zRankArgs, pArgs, p-pArgs);
      }
    }
  }

  if( rc!=SQLITE_OK ){

int sqlite3Fts5ConfigSetValue(
  Fts5Config *pConfig, 
  const char *zKey, 
  sqlite3_value *pVal,
  int *pbBadkey
){
  int rc = SQLITE_OK;




  if( 0==sqlite3_stricmp(zKey, "pgsz") ){
    int pgsz = 0;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      pgsz = sqlite3_value_int(pVal);
    }
    if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){
      *pbBadkey = 1;
    }else{
      pConfig->pgsz = pgsz;
    }
  }

  else if( 0==sqlite3_stricmp(zKey, "automerge") ){
    int nAutomerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      nAutomerge = sqlite3_value_int(pVal);
    }
    if( nAutomerge<0 || nAutomerge>64 ){
      *pbBadkey = 1;
    }else{
      if( nAutomerge==1 ) nAutomerge = FTS5_DEFAULT_AUTOMERGE;
      pConfig->nAutomerge = nAutomerge;
    }
  }

  else if( 0==sqlite3_stricmp(zKey, "crisismerge") ){
    int nCrisisMerge = -1;
    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
      nCrisisMerge = sqlite3_value_int(pVal);
    }
    if( nCrisisMerge<0 ){
      *pbBadkey = 1;
    }else{
      if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
      pConfig->nCrisisMerge = nCrisisMerge;
    }
  }

  else if( 0==sqlite3_stricmp(zKey, "rank") ){
    const char *zIn = (const char*)sqlite3_value_text(pVal);
    char *zRank;
    char *zRankArgs;
    rc = sqlite3Fts5ConfigParseRank(zIn, &zRank, &zRankArgs);
    if( rc==SQLITE_OK ){
      sqlite3_free(pConfig->zRank);
      sqlite3_free(pConfig->zRankArgs);
      pConfig->zRank = zRank;
      pConfig->zRankArgs = zRankArgs;
    }else if( rc==SQLITE_ERROR ){
      rc = SQLITE_OK;
      *pbBadkey = 1;
    }
  }else{
    *pbBadkey = 1;
  }
  return rc;
}

/*
** Load the contents of the %_config table into memory.
*/

  if( rc==SQLITE_OK ){
    while( SQLITE_ROW==sqlite3_step(p) ){
      const char *zK = (const char*)sqlite3_column_text(p, 0);
      sqlite3_value *pVal = sqlite3_column_value(p, 1);
      if( 0==sqlite3_stricmp(zK, "version") ){
        iVersion = sqlite3_value_int(pVal);
      }else{
        int bDummy = 0;
        sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, &bDummy);
      }
    }
    rc = sqlite3_finalize(p);
  }
  
  if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
    rc = SQLITE_ERROR;
    if( pConfig->pzErrmsg ){
      assert( 0==*pConfig->pzErrmsg );
      *pConfig->pzErrmsg = sqlite3_mprintf(

  va_end(ap);
}

static int fts5ExprIsspace(char t){
  return t==' ' || t=='\t' || t=='\n' || t=='\r';
}







/*
** Read the first token from the nul-terminated string at *pz.
*/
static int fts5ExprGetToken(
  Fts5Parse *pParse, 
  const char **pz,                /* IN/OUT: Pointer into buffer */
  Fts5Token *pToken

      pToken->n = (z2 - z);
      break;
    }

    default: {
      const char *z2;
      tok = FTS5_STRING;
      for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
      pToken->n = (z2 - z);
      if( pToken->n==2 && memcmp(pToken->p, "OR", 2)==0 )  tok = FTS5_OR;
      if( pToken->n==3 && memcmp(pToken->p, "NOT", 3)==0 ) tok = FTS5_NOT;
      if( pToken->n==3 && memcmp(pToken->p, "AND", 3)==0 ) tok = FTS5_AND;
      break;
    }
  }

      }else{
        segid = sqlite3_value_int(apVal[1]);
        height = sqlite3_value_int(apVal[2]);
        pgno = sqlite3_value_int(apVal[3]);
        iRowid = FTS5_SEGMENT_ROWID(segid, height, pgno);
        sqlite3_result_int64(pCtx, iRowid);
      }














    }else {
      sqlite3_result_error(pCtx, 
        "first arg to fts5_rowid() must be 'segment' "
        "or 'start-of-index'"
        , -1
      );
    }

      pRet->pGlobal = (Fts5Global*)pAux;
      pRet->eType = eType;
      pRet->db = db;
      pRet->zFts5Tbl = (char*)&pRet[1];
      pRet->zFts5Db = &pRet->zFts5Tbl[nTab];
      memcpy(pRet->zFts5Tbl, zTab, nTab);
      memcpy(pRet->zFts5Db, zDb, nDb);
      sqlite3Fts5Dequote(pRet->zFts5Tbl);
      sqlite3Fts5Dequote(pRet->zFts5Db);
    }
  }

  *ppVTab = (sqlite3_vtab*)pRet;
  return rc;
}

#
do_catchsql_test 11.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
  CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid);
} {1 {reserved fts5 column name: rowid}}

#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
  CREATE VIRTUAL TABLE t2 USING fts5(x,y);
} {}

}
do_execsql_test 15.1 {
  UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
}
do_catchsql_test 15.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}

#-------------------------------------------------------------------------
#
do_execsql_test 16.1 {
  CREATE VIRTUAL TABLE n1 USING fts5(a);
  INSERT INTO n1 VALUES('a b c d');
}

proc funk {} {
  set fd [db incrblob main n1_data block 10]
  fconfigure $fd -encoding binary -translation binary
  puts -nonewline $fd "\x44\x45"
  close $fd
  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
}
db func funk funk

do_catchsql_test 16.2 {
  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
} {1 {SQL logic error or missing database}}

finish_test

# 2015 Jan 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file focuses on the code in fts5_config.c, which is largely concerned
# with parsing the various configuration and CREATE TABLE options.
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5config

#-------------------------------------------------------------------------
# Try different types of quote characters.
#
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5('a', "b", [c], `d`);
  PRAGMA table_info = t1;
} {
  0 a {} 0 {} 0 
  1 b {} 0 {} 0 
  2 c {} 0 {} 0 
  3 d {} 0 {} 0
}

#-------------------------------------------------------------------------
# Syntax errors in the prefix= option.
#
foreach {tn opt} {
  1 {prefix=x}  
  2 {prefix='x'}
  3 {prefix='$'}
} {
  set res [list 1 {malformed prefix=... directive}]
  do_catchsql_test 2.$tn "CREATE VIRTUAL TABLE f1 USING fts5(x, $opt)" $res
}

#-------------------------------------------------------------------------
# Syntax errors in the 'rank' option.
#
foreach {tn val} {
  1 "f1(xyz)"
  2 "f1(zyx)"
  3 "f1(nzz)"
  4 "f1(x'!!')"
  5 "f1(x':;')"
  6 "f1(x'[]')"
  7 "f1(x'{}')"
  8 "f1('abc)"
} {
  do_catchsql_test 3.$tn {
    INSERT INTO t1(t1, rank) VALUES('rank', $val);
  } {1 {SQL logic error or missing database}}
}

#-------------------------------------------------------------------------
# The parsing of SQL literals specified as part of 'rank' options.
#
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE zzz USING fts5(one);
  INSERT INTO zzz VALUES('a b c');
}
proc first {cmd A} { return $A }
sqlite3_fts5_create_function db first first

foreach {tn arg} {
  1 "123"
  2 "'01234567890ABCDEF'"
  3 "x'0123'"
  4 "x'ABCD'"
  5 "x'0123456789ABCDEF'"
  6 "x'0123456789abcdef'"
  7 "22.5"
  8 "-91.5"
  9 "-.5"
  10 "''''"
  11 "+.5"
} {
  set func [string map {' ''} "first($arg)"]
  do_execsql_test 4.1.$tn "
    INSERT INTO zzz(zzz, rank) VALUES('rank', '$func');
    SELECT rank IS $arg FROM zzz WHERE zzz MATCH 'a + b + c'
  " 1
}

do_execsql_test 4.2 {
  INSERT INTO zzz(zzz, rank) VALUES('rank', 'f1()');
} {}

#-------------------------------------------------------------------------
# Misquoting in tokenize= and other options. 
#
do_catchsql_test 5.1 {
  CREATE VIRTUAL TABLE xx USING fts5(x, tokenize="porter 'ascii");
} {1 {parse error in tokenize directive}} 

breakpoint
do_catchsql_test 5.2 {
  CREATE VIRTUAL TABLE xx USING fts5(x, [y[]);
} {0 {}}

do_catchsql_test 5.3 {
  CREATE VIRTUAL TABLE yy USING fts5(x, [y]]);
} {1 {unrecognized token: "]"}}

#-------------------------------------------------------------------------
# Errors in prefix= directives.
#
do_catchsql_test 6.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix=1, prefix=2);
} {1 {multiple prefix=... directives}}
do_catchsql_test 6.2 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1, 2, 1001');
} {1 {prefix length out of range: 1001}}
do_catchsql_test 6.3 {
  CREATE VIRTUAL TAbLE abc USING fts5(a, prefix='1, 2, 0000');
} {1 {prefix length out of range: 0}}
do_catchsql_test 6.4 {
  CREATE VIRTUAL TABLE abc USING fts5(a, prefix='1  , 1000000');
} {1 {malformed prefix=... directive}}

#-------------------------------------------------------------------------
# Duplicate tokenize= and other options.
#
do_catchsql_test 7.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, tokenize=porter, tokenize=ascii);
} {1 {multiple tokenize=... directives}}
do_catchsql_test 7.2 {
  CREATE VIRTUAL TABLE abc USING fts5(a, content=porter, content=ascii);
} {1 {multiple content=... directives}}
do_catchsql_test 7.3 {
  CREATE VIRTUAL TABLE abc USING fts5(a, content_rowid=porter, content_rowid=a);
} {1 {multiple content_rowid=... directives}}

#-------------------------------------------------------------------------
# Unrecognized option.
#
do_catchsql_test 8.0 {
  CREATE VIRTUAL TABLE abc USING fts5(a, nosuchoption=123);
} {1 {unrecognized option: "nosuchoption"}}
do_catchsql_test 8.1 {
  CREATE VIRTUAL TABLE abc USING fts5(a, "nosuchoption"=123);
} {1 {parse error in ""nosuchoption"=123"}}

#-------------------------------------------------------------------------
# Errors in:
#
#   9.1.* 'pgsz' options.
#   9.2.* 'automerge' options.
#   9.3.* 'crisismerge' options.
#
do_execsql_test 9.0 {
  CREATE VIRTUAL TABLE abc USING fts5(a, b);
} {}
do_catchsql_test 9.1.1 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.1.2 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 50000000);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.1.3 {
  INSERT INTO abc(abc, rank) VALUES('pgsz', 66.67);
} {1 {SQL logic error or missing database}}

do_catchsql_test 9.2.1 {
  INSERT INTO abc(abc, rank) VALUES('automerge', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.2.2 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 50000000);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.2.3 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 66.67);
} {1 {SQL logic error or missing database}}
do_execsql_test 9.2.4 {
  INSERT INTO abc(abc, rank) VALUES('automerge', 1);
} {}

do_catchsql_test 9.3.1 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', -5);
} {1 {SQL logic error or missing database}}
do_catchsql_test 9.3.2 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 66.67);
} {1 {SQL logic error or missing database}}
do_execsql_test 9.3.3 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 1);
} {}
do_execsql_test 9.3.4 {
  INSERT INTO abc(abc, rank) VALUES('crisismerge', 50000000);
} {}

do_catchsql_test 9.4.1 {
  INSERT INTO abc(abc, rank) VALUES('nosuchoption', 1);
} {1 {SQL logic error or missing database}}

finish_test

  7 {(a OR b) NOT c)}              {fts5: syntax error near ")"}
  8 {nosuch: a nosuch2: b}         {no such column: nosuch}
  9 {addr: a nosuch2: b}           {no such column: nosuch2}
  10 {NOT}                          {fts5: syntax error near "NOT"}
  11 {a AND "abc}                  {unterminated string}

  12 {NEAR(a b, xyz)}              {expected integer, got "xyz"}
  13 {NEAR(a b, // )}              {fts5: syntax error near "/"}
} {
  do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
}

#-------------------------------------------------------------------------
# Experiment with a tokenizer that considers " to be a token character.
#

  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<10)
  INSERT INTO zzz SELECT rnddoc(10) || ' xccc' FROM ii;
}

do_faultsim_test 4.1 -faults oom-trans* -prep {
} -body {
  execsql { INSERT INTO zzz(zzz) VALUES('integrity-check') }
} -test {
  faultsim_test_result {0 {}}
}

#-------------------------------------------------------------------------
# OOM while parsing a tokenize=option
#
reset_db
faultsim_save_and_close
do_faultsim_test 5.0 -faults oom-* -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { 
    CREATE VIRTUAL TABLE uio USING fts5(a, b, 
      tokenize="porter 'ascii'",
      content="another table",
      content_rowid="somecolumn"
    );
  }
} -test {
  faultsim_test_result {0 {}}
}

finish_test

do_faultsim_test 10.3 -faults oom-t* -body {
  db one { SELECT fts5_expr('x:a', 'x') }
} -test {
  faultsim_test_result {0 {x : "a"}}
}

#-------------------------------------------------------------------------
# OOM while configuring 'rank' option.
#
reset_db
do_execsql_test 11.0 {
  CREATE VIRTUAL TABLE ft USING fts5(x);
}
do_faultsim_test 11.1 -faults oom-* -body {
  db eval { 
    INSERT INTO ft(ft, rank) VALUES('rank', 'bm25(10.0, 5.0)');
  }
} -test {
  faultsim_test_result {0 {}}
}

finish_test

    for {set i 0} {$i < 20} {incr i} {
      do_test 4.$tn.4.$i {
        execsql { INSERT INTO x8(x8, rank) VALUES('merge', 1); }
        mycount
      } $expect
      break
    }
#  db eval {SELECT fts5_decode(rowid, block) AS r FROM x8_data} { puts $r }
}

finish_test

    DELETE FROM t1;
    INSERT INTO t1 VALUES('$doc');
    SELECT count(*) FROM t1 WHERE t1 MATCH '$near';
  " $res]
}

execsql { 
  CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = "ascii tokenchars '.'") 
}

do_near_test 1.1 ". . a . . . b . ." { NEAR(a b, 5) } 1
do_near_test 1.2 ". . a . . . b . ." { NEAR(a b, 4) } 1
do_near_test 1.3 ". . a . . . b . ." { NEAR(a b, 3) } 1
do_near_test 1.4 ". . a . . . b . ." { NEAR(a b, 2) } 0

  INSERT INTO tt VALUES('f d c a c e', 'd g d e g d');
  INSERT INTO tt VALUES('g d e f a g x', 'f f d a a b');
  INSERT INTO tt VALUES('g c f b c g', 'a g f d c b');
  INSERT INTO tt VALUES('c e c f g b', 'f e d b g a');
  INSERT INTO tt VALUES('g d e f d e', 'a c d b a g');
  INSERT INTO tt VALUES('e f a c c b', 'b f e a f d y');
  INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
}


set res_col {
  a 0 6 11    a 1 7 9
  b 0 6 7     b 1 7 7 
  c 0 6 12    c 1 5 8 
  d 0 4 6     d 1 9 13 
  e 0 6 7     e 1 6 6 
  f 0 9 10    f 1 7 10 
  g 0 5 7     g 1 5 7
  x 0 1 1     y 1 1 1
}
set res_row {




  a 10 20   b 9 14   c 9 20   d 9 19   
  e 8 13   f 10 20   g 7 14   x 1 1   
  y 1 1
}

foreach {tn tbl resname} {
  1 "fts5vocab(tt, 'col')" res_col
  2 "fts5vocab(tt, 'row')" res_row
  3 "fts5vocab(tt, \"row\")" res_row
  4 "fts5vocab(tt, [row])" res_row
  5 "fts5vocab(tt, `row`)" res_row

  6 "fts5vocab('tt', 'row')" res_row
  7 "fts5vocab(\"tt\", \"row\")" res_row
  8 "fts5vocab([tt], [row])" res_row
  9 "fts5vocab(`tt`, `row`)" res_row
} {
  do_execsql_test 2.$tn "
    DROP TABLE IF EXISTS tv;
    CREATE VIRTUAL TABLE tv USING $tbl;
    SELECT * FROM tv;
  " [set $resname]
}

#-------------------------------------------------------------------------
#
foreach {tn sql} {
  1 { CREATE VIRTUAL TABLE aa USING fts5vocab() }
  2 { CREATE VIRTUAL TABLE aa USING fts5vocab(x) }
  3 { CREATE VIRTUAL TABLE aa USING fts5vocab(x,y,z) }