SQLite

  );
  if( p->bHasDocsize ){
    fts3DbExec(&rc, db, 
        "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);",
        p->zDb, p->zName
    );
  }
  assert( p->bHasStat==p->bFts4 );
  if( p->bHasStat ){
    sqlite3Fts3CreateStatTable(&rc, p);
  }
  return rc;
}

/*

  p->nColumn = nCol;
  p->nPendingData = 0;
  p->azColumn = (char **)&p[1];
  p->pTokenizer = pTokenizer;
  p->nMaxPendingData = FTS3_MAX_PENDING_DATA;
  p->bHasDocsize = (isFts4 && bNoDocsize==0);
  p->bHasStat = isFts4;
  p->bFts4 = isFts4;
  p->bDescIdx = bDescIdx;
  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  p->zContentTbl = zContent;
  p->zLanguageid = zLanguageid;
  zContent = 0;
  zLanguageid = 0;
  TESTONLY( p->inTransaction = -1 );

      return SQLITE_NOMEM;
    }

    pCsr->iLangid = 0;
    if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]);

    rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid,
        p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr
    );
    if( rc!=SQLITE_OK ){
      if( rc==SQLITE_ERROR ){
        static const char *zErr = "malformed MATCH expression: [%s]";
        p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery);
      }
      return rc;

  int nToken = 0;
  int nOr = 0;

  /* Allocate a MultiSegReader for each token in the expression. */
  fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc);

  /* Determine which, if any, tokens in the expression should be deferred. */
  if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){
    Fts3TokenAndCost *aTC;
    Fts3Expr **apOr;
    aTC = (Fts3TokenAndCost *)sqlite3_malloc(
        sizeof(Fts3TokenAndCost) * nToken
      + sizeof(Fts3Expr *) * nOr * 2
    );
    apOr = (Fts3Expr **)&aTC[nToken];

  */
  sqlite3_stmt *aStmt[37];

  char *zReadExprlist;
  char *zWriteExprlist;

  int nNodeSize;                  /* Soft limit for node size */
  u8 bFts4;                       /* True for FTS4, false for FTS3 */
  u8 bHasStat;                    /* True if %_stat table exists */
  u8 bHasDocsize;                 /* True if %_docsize table exists */
  u8 bDescIdx;                    /* True if doclists are in reverse order */
  u8 bIgnoreSavepoint;            /* True to ignore xSavepoint invocations */
  int nPgsz;                      /* Page size for host database */
  char *zSegmentsTbl;             /* Name of %_segments table */
  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */

static int fts3MatchinfoCheck(
  Fts3Table *pTab, 
  char cArg,
  char **pzErr
){
  if( (cArg==FTS3_MATCHINFO_NPHRASE)
   || (cArg==FTS3_MATCHINFO_NCOL)
   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
   || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
   || (cArg==FTS3_MATCHINFO_LCS)
   || (cArg==FTS3_MATCHINFO_HITS)
  ){
    return SQLITE_OK;
  }
  *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);

){
  Fts3Table *p = (Fts3Table*)pCsr->base.pVtab;
  int nOvfl = 0;
  int ii;
  int rc = SQLITE_OK;
  int pgsz = p->nPgsz;

  assert( p->bFts4 );
  assert( pgsz>0 );

  for(ii=0; rc==SQLITE_OK && ii<pMsr->nSegment; ii++){
    Fts3SegReader *pReader = pMsr->apSegment[ii];
    if( !fts3SegReaderIsPending(pReader) 
     && !fts3SegReaderIsRootOnly(pReader) 
    ){

      }else{
        nEntry++;
        for(iCol=0; iCol<=p->nColumn; iCol++){
          aSzIns[iCol] += aSz[iCol];
        }
      }
    }
    if( p->bFts4 ){
      fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry);
    }
    sqlite3_free(aSz);

    if( pStmt ){
      int rc2 = sqlite3_finalize(pStmt);
      if( rc==SQLITE_OK ){

    nMin = fts3Getint(&z);
  }

  if( z[0]!='\0' || nMin<2 ){
    rc = SQLITE_ERROR;
  }else{
    rc = SQLITE_OK;
    if( !p->bHasStat ){
      assert( p->bFts4==0 );
      sqlite3Fts3CreateStatTable(&rc, p);
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3Fts3Incrmerge(p, nMerge, nMin);
    }
    sqlite3Fts3SegmentsClose(p);
  }
  return rc;
}

  Fts3Table *p,                   /* FTS3 table handle */
  const char *zParam              /* Nul-terminated string containing boolean */
){
  int rc = SQLITE_OK;
  sqlite3_stmt *pStmt = 0;
  p->bAutoincrmerge = fts3Getint(&zParam)!=0;
  if( !p->bHasStat ){
    assert( p->bFts4==0 );
    sqlite3Fts3CreateStatTable(&rc, p);
    if( rc ) return rc;
  }
  rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0);
  if( rc ) return rc;;
  sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE);
  sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge);

    }
    if( p->bHasDocsize ){
      fts3InsertDocsize(&rc, p, aSzIns);
    }
    nChng++;
  }

  if( p->bFts4 ){
    fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng);
  }

 update_out:
  sqlite3_free(aSzIns);
  sqlite3Fts3SegmentsClose(p);
  return rc;

#   fts3_doclist TBL TERM WHERE
#     Todo.
#
#
#

#-------------------------------------------------------------------------
# USAGE: fts3_build_db_1 SWITCHES N
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t1".
#
proc fts3_build_db_1 {args} {

  set default(-module) fts4

  set nArg [llength $args]
  if {($nArg%2)==0} {
    error "wrong # args: should be \"fts3_build_db_1 ?switches? n\""
  }

  set n [lindex $args [expr $nArg-1]]
  array set opts [array get default]
  array set opts [lrange $args 0 [expr $nArg-2]]
  foreach k [array names opts] {
    if {0==[info exists default($k)]} { error "unknown option: $k" }
  }

  if {$n > 10000} {error "n must be <= 10000"}
  db eval "CREATE VIRTUAL TABLE t1 USING $opts(-module) (x, y)"

  set xwords [list zero one two three four five six seven eight nine ten]
  set ywords [list alpha beta gamma delta epsilon zeta eta theta iota kappa]

  for {set i 0} {$i < $n} {incr i} {
    set x ""
    set y ""

#-------------------------------------------------------------------------
# USAGE: fts3_build_db_2 N ARGS
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t2".
#
proc fts3_build_db_2 {args} {

  set default(-module) fts4
  set default(-extra)   ""

  set nArg [llength $args]
  if {($nArg%2)==0} {
    error "wrong # args: should be \"fts3_build_db_1 ?switches? n\""
  }

  set n [lindex $args [expr $nArg-1]]
  array set opts [array get default]
  array set opts [lrange $args 0 [expr $nArg-2]]
  foreach k [array names opts] {
    if {0==[info exists default($k)]} { error "unknown option: $k" }
  }

  if {$n > 100000} {error "n must be <= 100000"}

  set sql "CREATE VIRTUAL TABLE t2 USING $opts(-module) (content"
  if {$opts(-extra) != ""} {
    append sql ", " $opts(-extra)
  }
  append sql ")"
  db eval $sql

  set chars [list a b c d e f g h  i j k l m n o p  q r s t u v w x  y z ""]

  for {set i 0} {$i < $n} {incr i} {
    set word ""

#   2.1: Run the integrity check code to check it passes.
#
#   2.2: Make a series of minor changes to the underlying FTS data structures
#        (e.g. delete or insert a row from the %_content table). Check that
#        this causes the integrity-check code to fail.
#

do_test 2.0 { fts3_build_db_2 -extra {prefix="3,1"} 20000 } {}
do_test 2.1 { fts_integrity db t2 } {ok}
foreach {tn disruption} {
  1 {
    INSERT INTO t2_content VALUES(NULL, 'xyz')
  }
  3 {
    DELETE FROM t2_segdir WHERE level=0 AND idx=(

} {
  do_execsql_test  3.2.1.$tn "BEGIN; $disruption"
  do_catchsql_test 3.2.2.$tn {
    INSERT INTO t3 (t3) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  3.2.3.$tn "ROLLBACK"
}



finish_test

# This file implements regression tests for SQLite library.  The
# focus of this script is testing the incremental merge function.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/fts3_common.tcl


# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

proc fts3_integrity_check {tbl} {
  db eval "INSERT INTO $tbl ($tbl) VALUES('integrity-check')"
  return "ok"
}

foreach mod {fts3 fts4} {
  set ::testprefix fts4merge-$mod
  reset_db

  #-------------------------------------------------------------------------
  # Test cases 1.*
  #
  do_test 1.0 { fts3_build_db_1 -module $mod 1004 } {}
  do_test 1.1 { fts3_integrity_check t1 } {ok}
  do_execsql_test 1.1 { 
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
  } {
    0 {0 1 2 3 4 5 6 7 8 9 10 11} 
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13}
    2 {0 1 2}
  }
  
  for {set i 0} {$i<20} {incr i} {
    do_execsql_test 1.2.$i.1 { INSERT INTO t1(t1) VALUES('merge=1') }
    do_test 1.2.$i.2 { fts3_integrity_check t1 } ok
    do_execsql_test 1.2.$i.3 { 
      SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
    } {123 132 213 231 312 321}
  }
  
  do_execsql_test 1.3 { 
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
  } {
    0 {0 1 2 3} 
    1 {0 1 2 3 4 5 6} 
    2 {0 1 2 3}
  }
  
  for {set i 0} {$i<100} {incr i} {
    do_execsql_test 1.4.$i { INSERT INTO t1(t1) VALUES('merge=1,4') }
    do_test 1.4.$i.2 { fts3_integrity_check t1 } ok
    do_execsql_test 1.4.$i.3 { 
      SELECT docid FROM t1 WHERE t1 MATCH 'zero one two three'
    } {123 132 213 231 312 321}
  }
  
  do_execsql_test 1.5 { 
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level 
  } {
    2 {0 1}
    3 0
  }
  
  #-------------------------------------------------------------------------
  # Test cases 2.* test that errors in the xxx part of the 'merge=xxx' are
  # handled correctly.
  #
  do_execsql_test 2.0 "CREATE VIRTUAL TABLE t2 USING $mod"
  
  foreach {tn arg} {
    1   {merge=abc}
    2   {merge=%%%}
    3   {merge=,}
    4   {merge=5,}
    5   {merge=6,%}
    6   {merge=6,six}
    7   {merge=6,1}
    8   {merge=6,0}
  } {
    do_catchsql_test 2.$tn { 
      INSERT INTO t2(t2) VALUES($arg);
    } {1 {SQL logic error or missing database}}
  }
  
  #-------------------------------------------------------------------------
  # Test cases 3.*
  #
  do_test 3.0 { 
    reset_db
    execsql { PRAGMA page_size = 512 }
    fts3_build_db_2 -module $mod 30040 
  } {}
  do_test 3.1 { fts3_integrity_check t2 } {ok}
  
  do_execsql_test 3.2 { 
    SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
  } {
    0 {0 1 2 3 4 5 6} 
    1 {0 1 2 3 4} 
    2 {0 1 2 3 4} 
    3 {0 1 2 3 4 5 6}
  }
  
  do_execsql_test 3.3 { 
    INSERT INTO t2(t2) VALUES('merge=1000000,2');
    SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level 
  } {
    0 0 
    2 0
    3 0 
    4 0
    6 0
  }
  
  #-------------------------------------------------------------------------
  # Test cases 4.*
  #
  reset_db
  do_execsql_test 4.1 "
    PRAGMA page_size = 512;
    CREATE VIRTUAL TABLE t4 USING $mod;
    PRAGMA main.page_size;
  " {512}
  
  do_test 4.2 {
    foreach x {a c b d e f g h i j k l m n o p} {
      execsql "INSERT INTO t4 VALUES('[string repeat $x 600]')"
    }
    execsql {SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level}
  } {0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15}}
  
  foreach {tn expect} {
    1  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 1 0"
    2  "0 {0 1 2 3 4 5 6 7 8 9 10 11 12}    1 0"
    3  "0 {0 1 2 3 4 5 6 7 8 9 10 11}       1 0"
    4  "0 {0 1 2 3 4 5 6 7 8 9 10}          1 0"
    5  "0 {0 1 2 3 4 5 6 7 8 9}             1 0"
    6  "0 {0 1 2 3 4 5 6 7 8}               1 0"
    7  "0 {0 1 2 3 4 5 6 7}                 1 0"
    8  "0 {0 1 2 3 4 5 6}                   1 0"
    9  "0 {0 1 2 3 4 5}                     1 0"
  } {
    do_execsql_test 4.3.$tn {
      INSERT INTO t4(t4) VALUES('merge=1,16');
      SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
    } $expect
  }
  
  do_execsql_test 4.4.1 {
    SELECT quote(value) FROM t4_stat WHERE rowid=1
  } {X'0006'}
  
  do_execsql_test 4.4.2 {
    DELETE FROM t4_stat WHERE rowid=1;
    INSERT INTO t4(t4) VALUES('merge=1,12');
    SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
  } "0 {0 1 2 3 4 5}                     1 0"
  
  
  #-------------------------------------------------------------------------
  # Test cases 5.*
  #
  # Test that if a crisis-merge occurs that disrupts an ongoing incremental
  # merge, the next call to "merge=A,B" identifies this and starts a new
  # incremental merge. There are two scenarios:
  #
  #   * There are less segments on the input level that the disrupted
  #     incremental merge operated on, or
  #   
  #   * Sufficient segments exist on the input level but the segments 
  #     contain keys smaller than the largest key in the potential output 
  #     segment.
  # 
  do_test 5.1 {
    reset_db
    fts3_build_db_1 -module $mod 1000
  } {}
  
  do_execsql_test 5.2 {
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  } {
    0 {0 1 2 3 4 5 6 7} 
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
    2 {0 1 2}
  }
  
  do_execsql_test 5.3 {
    INSERT INTO t1(t1) VALUES('merge=1,5');
    INSERT INTO t1(t1) VALUES('merge=1,5');
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
  } {
    0 {0 1 2}
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14} 
    2 {0 1 2 3}
  }
  
  do_execsql_test 5.4 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
  do_test 5.5 {
    foreach docid [execsql {SELECT docid FROM t1}] {
      execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
    }
  } {}
  
  do_execsql_test 5.6 {SELECT quote(value) from t1_stat WHERE rowid=1} {X'0105'}
  
  do_execsql_test 5.7 {
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
    SELECT quote(value) from t1_stat WHERE rowid=1;
  } {
    0 {0 1 2 3 4 5 6 7 8 9 10} 
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12} 
    2 {0 1 2 3 4 5 6 7}
    X'0105'
  }
  
  do_execsql_test 5.8 {
    INSERT INTO t1(t1) VALUES('merge=1,6');
    INSERT INTO t1(t1) VALUES('merge=1,6');
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
    SELECT quote(value) from t1_stat WHERE rowid=1;
  } {
    0 {0 1 2 3 4} 
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
    2 {0 1 2 3 4 5 6 7 8} X'0106'
  }
  
  do_test 5.8.1 { fts3_integrity_check t1 } ok
  
  do_test 5.9 {
    set L [expr 16*16*7 + 16*3 + 12]
    foreach docid [execsql {
        SELECT docid FROM t1 UNION ALL SELECT docid FROM t1 LIMIT $L
    }] {
      execsql {INSERT INTO t1 SELECT * FROM t1 WHERE docid=$docid}
    }
  } {}
  
  do_execsql_test 5.10 {
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
    SELECT quote(value) from t1_stat WHERE rowid=1;
  } {
    0 0 1 {0 1} 2 0 3 0 X'0106'
  }
  
  do_execsql_test 5.11 {
    INSERT INTO t1(t1) VALUES('merge=1,6');
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level;
    SELECT quote(value) from t1_stat WHERE rowid=1;
  } {
    0 0 1 {0 1} 2 0 3 0 X''
  }
  
  #-------------------------------------------------------------------------
  # Test cases 6.*
  #
  # At one point the following test caused an assert() to fail (because the
  # second 'merge=1,2' operation below actually "merges" a single input
  # segment, which was unexpected).
  #
  do_test 6.1 {
    reset_db
    set a [string repeat a 900]
    set b [string repeat b 900]
    set c [string repeat c 900]
    set d [string repeat d 900]

    execsql "CREATE VIRTUAL TABLE t1 USING $mod"
    execsql {
      BEGIN;
        INSERT INTO t1 VALUES($a);
        INSERT INTO t1 VALUES($b);
      COMMIT;
      BEGIN;
        INSERT INTO t1 VALUES($c);
        INSERT INTO t1 VALUES($d);
      COMMIT;
    }
  
    execsql {
      INSERT INTO t1(t1) VALUES('merge=1,2');
      INSERT INTO t1(t1) VALUES('merge=1,2');
    }
  } {}
  
  #-------------------------------------------------------------------------
  # Test cases 7.*
  #
  # Test that the value returned by sqlite3_total_changes() increases by
  # 1 following a no-op "merge=A,B", or by more than 1 if actual work is
  # performed.
  #
  do_test 7.0 {
    reset_db
    fts3_build_db_1 -module $mod 1000
  } {}
  
  do_execsql_test 7.1 {
    SELECT level, group_concat(idx, ' ') FROM t1_segdir GROUP BY level
  } {
    0 {0 1 2 3 4 5 6 7} 
    1 {0 1 2 3 4 5 6 7 8 9 10 11 12 13} 
    2 {0 1 2}
  }
  do_test 7.2 {
    set x [db total_changes]
    execsql { INSERT INTO t1(t1) VALUES('merge=2,10') }
    expr { ([db total_changes] - $x)>1 }
  } {1}
  do_test 7.3 {
    set x [db total_changes]
    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
    expr { ([db total_changes] - $x)>1 }
  } {1}
  do_test 7.4 {
    set x [db total_changes]
    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
    expr { ([db total_changes] - $x)>1 }
  } {0}
  do_test 7.5 {
    set x [db total_changes]
    execsql { INSERT INTO t1(t1) VALUES('merge=200,10') }
    expr { ([db total_changes] - $x)>1 }
  } {0}

}

finish_test