SQLite

        if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
          if( (*p2&0xFE)==0 ) break;
          fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
        }else{
          if( (*p1&0xFE)==0 ) break;
          fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
        }
      }

      if( pSave ){
        assert( pp && p );
        p = pSave;
      }

      fts3ColumnlistCopy(0, &p1);
      fts3ColumnlistCopy(0, &p2);
      assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 );
      if( 0==*p1 || 0==*p2 ) break;

       || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE
       || mergetype==MERGE_NEAR   || mergetype==MERGE_POS_NEAR
  );

  if( !aBuffer ){
    return SQLITE_NOMEM;
  }





  /* Read the first docid from each doclist */
  fts3GetDeltaVarint2(&p1, pEnd1, &i1);
  fts3GetDeltaVarint2(&p2, pEnd2, &i2);

  switch( mergetype ){
    case MERGE_OR:

    }

    default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); {
      char *aTmp = 0;
      char **ppPos = 0;
      if( mergetype==MERGE_POS_NEAR ){
        ppPos = &p;
        aTmp = sqlite3_malloc(2*(n1+n2+1));
        if( !aTmp ){
          return SQLITE_NOMEM;
        }
      }

      while( p1 && p2 ){
        if( i1==i2 ){

  assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) );
  assert( nVal==0 || nVal==1 );
  assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) );

  /* In case the cursor has been used before, clear it now. */
  sqlite3_finalize(pCsr->pStmt);
  sqlite3_free(pCsr->aDoclist);
  sqlite3Fts3ExprFree(pCsr->pExpr);
  memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));

  /* Compile a SELECT statement for this cursor. For a full-table-scan, the
  ** statement loops through all rows of the %_content table. For a
  ** full-text query or docid lookup, the statement retrieves a single
  ** row by docid.
  */

** function is called by the sqlite3_extension_init() entry point.
*/
int sqlite3Fts3Init(sqlite3 *db){
  int rc = SQLITE_OK;
  Fts3Hash *pHash = 0;
  const sqlite3_tokenizer_module *pSimple = 0;
  const sqlite3_tokenizer_module *pPorter = 0;

#ifdef SQLITE_ENABLE_ICU
  const sqlite3_tokenizer_module *pIcu = 0;
  sqlite3Fts3IcuTokenizerModule(&pIcu);
#endif

  sqlite3Fts3SimpleTokenizerModule(&pSimple);
  sqlite3Fts3PorterTokenizerModule(&pPorter);




  /* Allocate and initialise the hash-table used to store tokenizers. */
  pHash = sqlite3_malloc(sizeof(Fts3Hash));
  if( !pHash ){
    rc = SQLITE_NOMEM;
  }else{
    sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1);
  }

  /* Load the built-in tokenizers into the hash table */
  if( rc==SQLITE_OK ){
    if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
     || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
#ifdef SQLITE_ENABLE_ICU
     || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
#endif
    ){
      rc = SQLITE_NOMEM;
    }
  }

#ifdef SQLITE_TEST
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts3ExprInitTestInterface(db);
  }
#endif

  /* Create the virtual table wrapper around the hash-table and overload 
  ** the two scalar functions. If this is successful, register the
  ** module with sqlite.
  */
  if( SQLITE_OK==rc 

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif

#endif /* _FTSINT_H */

  sqlite3_free(azCol);
}

/*
** Register the query expression parser test function fts3_exprtest() 
** with database connection db. 
*/
int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
  return sqlite3_create_function(
      db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
  );
}

#endif
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */

const char *sqlite3Fts3NextToken(const char *zStr, int *pn){
  const char *z1;
  const char *z2 = 0;

  /* Find the start of the next token. */
  z1 = zStr;
  while( z2==0 ){
    char c = *z1;
    switch( c ){
      case '\0': return 0;        /* No more tokens here */
      case '\'':
      case '"':
      case '`': {
        z2 = z1;
        while( *++z2 && (*z2!=c || *++z2==c) );

        break;
      }
      case '[':
        z2 = &z1[1];
        while( *z2 && z2[0]!=']' ) z2++;
        if( *z2 ) z2++;
        break;

    rc = sqlite3Fts3PendingTermsFlush(p);
    if( rc==SQLITE_OK ){
      rc = fts3SegmentMerge(p, -1);
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
    }else{
      sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0);
      sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0);
    }
  }
  return rc;
}

#endif

  pParse = sqlite3StackAllocZero(db, sizeof(*pParse));
  if( pParse==0 ){
    rc = SQLITE_NOMEM;
  }else{
    pParse->declareVtab = 1;
    pParse->db = db;
  

    if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) 
     && !db->mallocFailed
     && pParse->pNewTable
     && !pParse->pNewTable->pSelect
     && (pParse->pNewTable->tabFlags & TF_Virtual)==0
    ){
      if( !pTab->aCol ){
        pTab->aCol = pParse->pNewTable->aCol;
        pTab->nCol = pParse->pNewTable->nCol;
        pParse->pNewTable->nCol = 0;
        pParse->pNewTable->aCol = 0;
      }
      db->pVTab = 0;
    }else{
      sqlite3Error(db, SQLITE_ERROR, zErr);
      sqlite3DbFree(db, zErr);
      rc = SQLITE_ERROR;
    }
    pParse->declareVtab = 0;
  
    if( pParse->pVdbe ){

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# If this build does not include FTS3, skip the tests in this file.
#
ifcapable !fts3 { finish_test ; return }
source $testdir/fts3_common.tcl
source $testdir/malloc_common.tcl

# Procs used to make the tests in this file easier to read.
#
proc ddl_test {tn ddl} {
  uplevel [list do_write_test e_fts3-$tn sqlite_master $ddl]
}
proc write_test {tn tbl sql} {
  uplevel [list do_write_test e_fts3-$tn $tbl $sql]
}
proc read_test {tn sql result} {
  uplevel [list do_select_test e_fts3-$tn $sql $result]
}
proc error_test {tn sql result} {
  uplevel [list do_error_test e_fts3-$tn $sql $result]
}


#-------------------------------------------------------------------------
# The body of the following [foreach] block contains test cases to verify
# that the example code in fts3.html works as expected. The tests run three
# times, with different values for DO_MALLOC_TEST.
# 
#   DO_MALLOC_TEST=0: Run tests with no OOM errors.
#   DO_MALLOC_TEST=1: Run tests with transient OOM errors.
#   DO_MALLOC_TEST=2: Run tests with persistent OOM errors.
#
foreach {DO_MALLOC_TEST enc} {
  0 utf-8
  1 utf-8
  2 utf-8
  1 utf-16
} {

#if {$DO_MALLOC_TEST} break

# Reset the database and database connection. If this iteration of the 
# [foreach] loop is testing with OOM errors, disable the lookaside buffer.
#
db close
file delete -force test.db test.db-journal
sqlite3 db test.db
if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 }
db eval "PRAGMA encoding = '$enc'"

##########################################################################
# Test the example CREATE VIRTUAL TABLE statements in section 1.1 
# of fts3.in.
#
ddl_test   1.1.1.1 {CREATE VIRTUAL TABLE data USING fts3()}
read_test  1.1.1.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0}

} {illegal first argument to snippet}
error_test 2.1.7 {
  SELECT snippet() FROM t1 WHERE a MATCH 'one'
} {unable to use function snippet in the requested context}
error_test 2.1.8 {
  SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
#
do_malloc_test e_fts3-3 -tclbody { 
  if {[catch {sqlite3 db test.db}]} { error "out of memory" }
}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Verify the return values of the optimize() function. If no error occurs,
# the returned value should be "Index optimized" if the data structure
# was modified, or "Index already optimal" if it were not.
#
set DO_MALLOC_TEST 0
ddl_test   4.1 { CREATE VIRTUAL TABLE t4 USING fts3(a, b) }
write_test 4.2 t4_content {
  INSERT INTO t4 VALUES('In Xanadu', 'did Kubla Khan');
}
write_test 4.3 t4_content {
  INSERT INTO t4 VALUES('a stately pleasure', 'dome decree');
}
do_test e_fts3-4.4 {
  execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } 
} {{Index optimized}}
do_test e_fts3-4.5 {
  execsql { SELECT optimize(t4) FROM t4 LIMIT 1 } 
} {{Index already optimal}}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test that the snippet function appears to work correctly with 1, 2, 3
# or 4 arguments passed to it.
#
set DO_MALLOC_TEST 0
ddl_test   5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
  INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea.  So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 { 
  SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <b>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.4 { 
  SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <i>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.5 { 
  SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.6 { 
  SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXX Down to a sunless sea.  So twice five <i>miles</i> of fertile ground With walls and towers were XXX}}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0
ddl_test   6.1 { CREATE VIRTUAL TABLE t6 USING fts3(x) }
write_test 6.2 t5_content { INSERT INTO t6 VALUES('a'); }
write_test 6.3 t5_content { INSERT INTO t6 VALUES('b'); }
write_test 6.4 t5_content { INSERT INTO t6 VALUES('c'); }
read_test  6.5 { SELECT * FROM t6 WHERE t6 MATCH '' } {}
read_test  6.6 { SELECT * FROM t6 WHERE x MATCH '' } {}
read_test  6.7 { SELECT * FROM t6 WHERE t6 MATCH NULL } {}
read_test  6.8 { SELECT * FROM t6 WHERE x MATCH NULL } {}
#-------------------------------------------------------------------------


#-------------------------------------------------------------------------
# Test a few facets of the FTS3 xFilter() callback implementation:
#
#   1. That the sqlite3_index_constraint.usable flag is respected.
#
#   2. That it is an error to use the "docid" or "rowid" column of
#      an FTS3 table as the LHS of a MATCH operator.
#
#   3. That it is an error to AND together two MATCH expressions in 
#      that refer to a single FTS3 table in a WHERE clause.
#
#
ddl_test   7.1.1 { CREATE VIRTUAL TABLE t7 USING fts3(a) }
ddl_test   7.1.2 { CREATE VIRTUAL TABLE t8 USING fts3(b) }
write_test 7.1.3 t7_content { INSERT INTO t7(docid, a) VALUES(4,'number four') }
write_test 7.1.4 t7_content { INSERT INTO t7(docid, a) VALUES(5,'number five') }
write_test 7.1.5 t8_content { INSERT INTO t8(docid, b) VALUES(4,'letter D') }
write_test 7.1.6 t8_content { INSERT INTO t8(docid, b) VALUES(5,'letter E') }
read_test  7.1.7 {
  SELECT a || ':' || b FROM t7 JOIN t8 USING(docid)
} {{number four:letter D} {number five:letter E}}

error_test 7.2.1 {
  SELECT * FROM t7 WHERE docid MATCH 'number'
} {unable to use function MATCH in the requested context}
error_test 7.2.2 {
  SELECT * FROM t7 WHERE rowid MATCH 'number'
} {unable to use function MATCH in the requested context}

error_test 7.3.1 {
  SELECT * FROM t7 WHERE a MATCH 'number' AND a MATCH 'four'
} {unable to use function MATCH in the requested context}
error_test 7.3.2 {
  SELECT * FROM t7, t8 WHERE a MATCH 'number' AND a MATCH 'four'
} {unable to use function MATCH in the requested context}
error_test 7.3.3 {
  SELECT * FROM t7, t8 WHERE b MATCH 'letter' AND b MATCH 'd'
} {unable to use function MATCH in the requested context}
read_test 7.3.4 {
  SELECT * FROM t7, t8 WHERE a MATCH 'number' AND b MATCH 'letter'
} {{number four} {letter D} {number four} {letter E} {number five} {letter D} {number five} {letter E}}

#-------------------------------------------------------------------------
# Test the quoting of FTS3 table column names. Names may be quoted using
# any of "", '', ``` or [].
#
ddl_test  8.1.1 { CREATE VIRTUAL TABLE t9a USING fts3("c1", [c2]) }
ddl_test  8.1.2 { CREATE VIRTUAL TABLE t9b USING fts3('c1', `c2`) }
read_test 8.1.3 { PRAGMA table_info(t9a) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0}
read_test 8.1.4 { PRAGMA table_info(t9b) } {0 c1 {} 0 {} 0 1 c2 {} 0 {} 0}
ddl_test  8.2.1 { CREATE VIRTUAL TABLE t9c USING fts3("c""1", 'c''2') }
read_test 8.2.2 { PRAGMA table_info(t9c) } {0 c\"1 {} 0 {} 0 1 c'2 {} 0 {} 0}
#-------------------------------------------------------------------------

#-------------------------------------------------------------------------
# Test that FTS3 tables can be renamed using the ALTER RENAME command.
# OOM errors are tested during ALTER RENAME commands also.
#
foreach DO_MALLOC_TEST {0 1 2} {
  db close
  file delete -force test.db test.db-journal
  sqlite3 db test.db
  if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 }

  ddl_test   9.1.1             { CREATE VIRTUAL TABLE t10 USING fts3(x) }
  write_test 9.1.2 t10_content { INSERT INTO t10 VALUES('fts3 tables') }
  write_test 9.1.3 t10_content { INSERT INTO t10 VALUES('are renameable') }

  read_test  9.1.4 {
    SELECT * FROM t10 WHERE t10 MATCH 'table*'
  } {{fts3 tables}}
  read_test  9.1.5 {
    SELECT * FROM t10 WHERE x MATCH 'rename*'
  } {{are renameable}}

  ddl_test   9.1.6             { ALTER TABLE t10 RENAME TO t11 }

  read_test  9.1.7 {
    SELECT * FROM t11 WHERE t11 MATCH 'table*'
  } {{fts3 tables}}
  read_test  9.1.8 {
    SELECT * FROM t11 WHERE x MATCH 'rename*'
  } {{are renameable}}
}


finish_test