SQLite

  fts5_level_segs t1
} {1}
db_save

do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
set segid [lindex [fts5_level_segids t1] 0]

sqlite3_db_config db DEFENSIVE 0
do_test 1.3 {
  execsql {
    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
  }
  catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}

do_test 1.4 {
  db_restore_and_reopen
  sqlite3_db_config db DEFENSIVE 0
  execsql {
    UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    rowid = fts5_rowid('segment', $segid, 4);
  }
  catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}

  INSERT INTO t3 VALUES('three o');
  INSERT INTO t3 VALUES('four e');
  INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}
sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}

finish_test

# Also tested is that "MATCH 'x*'" does not crash and sometimes reports
# corruption. It may not report the db as corrupt because truncating the
# final leaf to some sizes may create a valid leaf page.
#
set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] 
set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
set all [db eval {SELECT rowid FROM t1}]
sqlite3_db_config db DEFENSIVE 0
for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
  do_execsql_test 2.$i.1 {
    BEGIN;
      UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
  }

  do_catchsql_test 2.$i.2 {

    execsql ROLLBACK
  }
}

#--------------------------------------------------------------------
reset_db
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE x5 USING fts5(tt);
  INSERT INTO x5 VALUES('a');
  INSERT INTO x5 VALUES('a a');
  INSERT INTO x5 VALUES('a a a');
  INSERT INTO x5 VALUES('a a a a');

  set rowid [db one {
    SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
  }]
  set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
  set {} {}
} {} 

sqlite3_db_config db DEFENSIVE 0
for {set i 0} {$i < $L} {incr i} {
  do_test 1.2.$i {
    catchsql {
      BEGIN;
      UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
      INSERT INTO t1(t1) VALUES('integrity-check');
    }

#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}
sqlite3_db_config db DEFENSIVE 0

do_execsql_test 3.1 {
  SELECT count(*) FROM t1_data;
} {105}

proc do_3_test {tn} {
  set i 0

do_3_test 3.10

#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
sqlite3_db_config db DEFENSIVE 0
do_test 4.0 {
  execsql { 
    CREATE VIRTUAL TABLE t1 USING fts5(x);
    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  }
  for {set i 0} {$i < 100} {incr i} {
    set rnd [expr int(rand() * 100)]

  set end [lindex $var end]
  if {$end<=$i} break
  lset var end [expr $end - $i]
  set struct [binary format c* $var]

  db close
  sqlite3 db test.db
  sqlite3_db_config db DEFENSIVE 0

  db eval {
    BEGIN;
    UPDATE t1_data SET block = $struct WHERE id=10;
  }
  do_test 4.1.$i {
    incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]

    catch { db eval ROLLBACK }
  }
}

#------------------------------------------------------------------------
#
reset_db
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1 VALUES('bbbbb ccccc');
  SELECT quote(block) FROM t1_data WHERE rowid>100;
} {X'000000180630626262626201020201056363636363010203040A'}
do_execsql_test 6.1.1 {
  UPDATE t1_data SET block = 
  X'000000180630626262626201020201056161616161010203040A'
  WHERE rowid>100;
}
do_catchsql_test 6.1.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}

#-------
reset_db
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.2.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
  INSERT INTO t1 VALUES('aa bb cc dd ee');
  SELECT pgno, quote(term) FROM t1_idx;
} {2 X'' 4 X'3064'}
do_execsql_test 6.2.1 {
  UPDATE t1_idx SET term = X'3065' WHERE pgno=4;
}
do_catchsql_test 6.2.2 {
  INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}

#-------
reset_db
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.3.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
  INSERT INTO t1 VALUES('abc abcdef abcdefghi');
  SELECT quote(block) FROM t1_data WHERE id>100;
}    {X'0000001C043061626301020204036465660102030703676869010204040808'}
do_execsql_test 6.3.1 {
  BEGIN;

    INSERT INTO t5 VALUES( rnddoc(10000) );
    INSERT INTO t5 VALUES( rnddoc(10000) );
    INSERT INTO t5 VALUES( rnddoc(10000) );
    INSERT INTO t5(t5) VALUES('optimize');
  }
} {}

sqlite3_db_config db DEFENSIVE 0
do_test 7.1 {
  foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
    db eval BEGIN  
    db eval {DELETE FROM t5_data WHERE rowid = $i}
    set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
    if {$r != "1 {database disk image is malformed}"} { error $r }
    db eval ROLLBACK  

#
reset_db
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t1 USING fts5(x, y);
  INSERT INTO t1 VALUES('one', 'two');
}

sqlite3_db_config db DEFENSIVE 0
do_test 9.1.1 {
  set    blob "12345678"    ;# cookie
  append blob "0105"        ;# 1 level, total of 5 segments
  append blob "06"          ;# write counter
  append blob "0002"        ;# first level has 0 segments merging, 2 other.
  append blob "450108"      ;# first segment
  execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"

  INSERT INTO aa(zz) VALUES('a');
  SELECT length(sz) FROM aa_docsize;
} {1 1 1 1 1}
do_execsql_test 4.1 { 
  INSERT INTO aa(aa) VALUES('integrity-check'); 
}

sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 4.2 { 
  BEGIN;
    UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
    INSERT INTO aa(aa) VALUES('integrity-check'); 
} {1 {database disk image is malformed}}

do_catchsql_test 4.3 { 

  INSERT INTO f1(f1) VALUES('rebuild');
} {}

do_execsql_test 1.4 {
  INSERT INTO f1(f1) VALUES('integrity-check');
} {}

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.5 {
  DELETE FROM f1_data;
} {}
sqlite3_db_config db DEFENSIVE 1

do_catchsql_test 1.6 {
  INSERT INTO f1(f1) VALUES('integrity-check');
} {1 {database disk image is malformed}}

do_execsql_test 1.7 {
  INSERT INTO f1(f1) VALUES('rebuild');

  DELETE FROM x1 WHERE (rowid%2);
}

set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.3 {
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 2.4 {
  UPDATE x1_data SET block = X'';
  SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
sqlite3_db_config db DEFENSIVE 1

do_execsql_test 2.5 {
  INSERT INTO x1(x1, rank) VALUES('pgsz', 1024);
  INSERT INTO x1(x1) VALUES('rebuild');
}

set res [db one {SELECT count(*) FROM x1_data}]

  SELECT * FROM t1_config WHERE k='version'
} {version 4}

do_execsql_test 1.3 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'a';
} {1}

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.4 {
  UPDATE t1_config set v=5 WHERE k='version';
}

do_test 1.5 {
  db close
  sqlite3 db test.db
  catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}

do_test 1.6 {
  db close
  sqlite3 db test.db
  catchsql { INSERT INTO t1 VALUES('x y z') }
} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}

do_test 1.7 {
  sqlite3_db_config db DEFENSIVE 0
  execsql { DELETE FROM t1_config WHERE k='version' }
  db close
  sqlite3 db test.db
  catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}


finish_test

  i a 1 1 i b 1 1 i c 1 1
}]
if {[detail_is_none]} { set resc [row_to_col $resr] }

do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 8.2 {
  PRAGMA writable_schema = 1;
  UPDATE sqlite_master 
  SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
  WHERE name = 'x1';
}
db close

  set e2 [db eval { EXPLAIN SELECT * FROM rrr ORDER BY term DESC }]
  expr [lsearch $e2 SorterSort]<0
} 0



finish_test

# The following block of tests - rtree8-2.* - test a couple of database
# corruption cases. In this case things are not corrupted at the b-tree
# level, but the contents of the various tables used internally by an
# r-tree table are inconsistent.
#
populate_t1 50
do_execsql_test rtree8-2.1.1 { SELECT max(nodeno) FROM t1_node } {5}
sqlite3_db_config db DEFENSIVE 0
do_execsql_test rtree8-2.1.2 { DELETE FROM t1_node } {}
sqlite3_db_config db DEFENSIVE 1
for {set i 1} {$i <= 50} {incr i} {
  do_catchsql_test rtree8-2.1.3.$i { 
    SELECT * FROM t1 WHERE id = $i 
  } {1 {database disk image is malformed}}
}
do_catchsql_test rtree8-2.1.4 { 
  SELECT * FROM t1
} {1 {database disk image is malformed}}
do_catchsql_test rtree8-2.1.5 { 
  DELETE FROM t1
} {1 {database disk image is malformed}}

do_execsql_test rtree8-2.1.6 { 
  DROP TABLE t1;
  CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
} {}


populate_t1 50
sqlite3_db_config db DEFENSIVE 0
do_execsql_test rtree8-2.2.1 {
  DELETE FROM t1_parent
} {}
sqlite3_db_config db DEFENSIVE 1
do_catchsql_test rtree8-2.2.2 {
  DELETE FROM t1 WHERE id=25
} {1 {database disk image is malformed}}
do_execsql_test rtree8-2.2.3 { 
  DROP TABLE t1;
  CREATE VIRTUAL TABLE t1 USING rtree_i32(id, x1, x2);
} {}

  execsql BEGIN
  for {set i 0} {$i < 500} {incr i} {
    set x2 [expr $i+5]
    set y2 [expr $i+5]
    execsql { INSERT INTO t1 VALUES($i, $i, $x2, $i, $y2) }
  }
  execsql COMMIT
  sqlite3_db_config db DEFENSIVE 0
}

proc truncate_node {nodeno nTrunc} {
  set blob [db one {SELECT data FROM t1_node WHERE nodeno=$nodeno}]
  if {$nTrunc<0} {set nTrunc "end-$nTrunc"}
  set blob [string range $blob 0 $nTrunc]
  db eval { UPDATE t1_node SET data = $blob WHERE nodeno=$nodeno }

#-------------------------------------------------------------------------
# Truncated blobs in the _node table.
#
create_t1
populate_t1
sqlite3 db test.db
sqlite3_db_config db DEFENSIVE 0
do_execsql_test rtreeA-7.100 { 
  UPDATE t1_node SET data=x'' WHERE rowid=1;
} {}
do_catchsql_test rtreeA-7.110 {
  SELECT * FROM t1 WHERE x1>0 AND x1<100 AND x2>0 AND x2<100;
} {1 {undersize RTree blobs in "t1_node"}}
do_test rtreeA-7.120 {
  sqlite3_extended_errcode db
} {SQLITE_CORRUPT_VTAB}


finish_test

    CREATE VIRTUAL TABLE r1 USING $module (id, x1, x2, y1, y2);
    INSERT INTO r1 VALUES(1,  5, 5, 5, 5);  --  3
    INSERT INTO r1 VALUES(2,  6, 6, 6, 6);  --  9
    INSERT INTO r1 VALUES(3,  7, 7, 7, 7);  -- 15
    INSERT INTO r1 VALUES(4,  8, 8, 8, 8);  -- 21
    INSERT INTO r1 VALUES(5,  9, 9, 9, 9);  -- 27
  "
  sqlite3_db_config db DEFENSIVE 0
}

setup_simple_db
do_execsql_test 2.1 { 
  SELECT rtreecheck('r1') 
} {ok}

do_execsql_test 3.1 { 
  CREATE VIRTUAL TABLE r2 USING rtree_i32(id, x1, x2);
  INSERT INTO r2 VALUES(2, -1*(1<<31), -1*(1<<31)+5);
  SELECT rtreecheck('r2') 
} {ok}

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 3.2 {
  BEGIN;
    UPDATE r2_node SET data = X'123456';
    SELECT rtreecheck('r2')!="ok";
} {1}

do_execsql_test 3.3 {

do_execsql_test 5.0 {
  CREATE VIRTUAL TABLE r3 USING rtree_i32(id, x1, x2, y1, y2);
  WITH x(i) AS (
    SELECT 1 UNION ALL SELECT i+1 FROM x WHERE i<1000
  )
  INSERT INTO r3 SELECT i, i, i, i, i FROM x;
}
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 5.1 {
  BEGIN;
    UPDATE r3_node SET data = set_int32(data, 3, 5000);
    UPDATE r3_node SET data = set_int32(data, 4, 5000);
    SELECT rtreecheck('r3')=='ok'
} 0
do_execsql_test 5.2 {
  ROLLBACK;
  BEGIN;
    UPDATE r3_node SET data = set_int32(data, 3, 0);
    UPDATE r3_node SET data = set_int32(data, 4, 0);
    SELECT rtreecheck('r3')=='ok'
} 0

finish_test

  if( ( IsVirtual(pTab) 
     && sqlite3GetVTable(db, pTab)->pMod->pModule->xUpdate==0 )
   || ( (pTab->tabFlags & TF_Readonly)!=0
     && sqlite3WritableSchema(db)==0
     && pParse->nested==0)
   || ( (pTab->tabFlags & TF_Shadow)!=0
     && (db->flags & SQLITE_Defensive)!=0
     && db->nVdbeExec==0
     && db->pVtabCtx==0)
  ){
    sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName);
    return 1;
  }

#ifndef SQLITE_OMIT_VIEW
  if( !viewOk && pTab->pSelect ){

  saved_flags = db->flags;
  saved_mDbFlags = db->mDbFlags;
  saved_nChange = db->nChange;
  saved_nTotalChange = db->nTotalChange;
  saved_mTrace = db->mTrace;
  db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks;
  db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum;
  db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder
                   | SQLITE_Defensive | SQLITE_CountRows);
  db->mTrace = 0;

  zDbMain = db->aDb[iDb].zDbSName;
  pMain = db->aDb[iDb].pBt;
  isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain));

  /* Attach the temporary database as 'vacuum_db'. The synchronous pragma

do_test corrupt2-2.1 {

  forcedelete corrupt.db
  forcedelete corrupt.db-journal
  forcecopy test.db corrupt.db

  sqlite3 db2 corrupt.db 
  sqlite3_db_config db2 DEFENSIVE 0
  execsql "
    $::presql
    CREATE INDEX a1 ON abc(a);
    CREATE INDEX a2 ON abc(b);
    PRAGMA writable_schema = 1;
    UPDATE sqlite_master 
      SET name = 'a3', sql = 'CREATE INDEX a3' || substr(sql, 16, 10000)

  array set A $args

  catch {db close}
  forcedelete corrupt.db
  forcedelete corrupt.db-journal

  sqlite3 db corrupt.db 
  sqlite3_db_config db DEFENSIVE 0
  db eval $::presql
  eval $A(-tclprep)
  db eval $A(-sqlprep)
  db close

  eval $A(-corrupt)

  return [expr $nRow*$pgsz]
}

#    fts3_zero_long_segments TABLE ?LIMIT?
#
proc fts3_zero_long_segments {tbl limit} {
  sqlite3_db_config db DEFENSIVE 0
  execsql " 
    UPDATE ${tbl}_segments 
    SET block = zeroblob(length(block)) 
    WHERE length(block)>$limit
  "
  sqlite3_db_config db DEFENSIVE 1
  return [db changes]
}


proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*

set ::testprefix fts3corrupt


# Test that a doclist with a length field that indicates that the doclist
# extends past the end of the node on which it resides is correctly identified
# as database corruption.
#
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts3;
  INSERT INTO t1 VALUES('hello');
} {}
do_test fts3corrupt-1.1 {
  set blob [db one {SELECT root from t1_segdir}]
  set blob [binary format a7ca* $blob 24 [string range $blob 8 end]]

   "ayjafsraz addjj agsj asejtziqws acatvhegu aoxdjqblsvv aekdmmbs aaobe"
   "abjjvzubkwt alczv ati awz auyxgcxeb aymjoym anqoukprtyt atwfhpmbooh"
   "ajfqz aethlgir aclcx aowlyvetby aproqm afjlqtkv anebfy akzrcpfrrvw"
   "aoledfotm aiwlfm aeejlaej anz abgbvk aktfn aayoh anpywgdvgz"
   "acvmldguld asdvz aqb aeomsyzyu aggylhprbdz asrfkwz auipybpsn agsnszzfb"
}

sqlite3_db_config db DEFENSIVE 0
do_test fts3corrupt2-1.0 {
  execsql BEGIN
  execsql { CREATE VIRTUAL TABLE t2 USING FTS3(a, b); }
  execsql { INSERT INTO t2(t2) VALUES('nodesize=32') }
  foreach d $data {
    execsql { INSERT INTO t2 VALUES($d, $d) }
  }

    INSERT INTO t1 VALUES('one');
    INSERT INTO t1 VALUES('one');
  COMMIT;
}
do_execsql_test 1.1 {
  SELECT quote(root) from t1_segdir;
} {X'00036F6E6509010200010200010200'}
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.2 {
  UPDATE t1_segdir SET root = X'00036F6E650EFFFFFFFFFFFFFFFFFFFFFFFF0200';
}
do_catchsql_test 1.3 {
  SELECT rowid FROM t1 WHERE t1 MATCH 'one'
} {0 -1}

proc blob {a} { binary decode hex $a }
db func blob blob

do_execsql_test 1.1 {
  SELECT quote(root) FROM ft_segdir;
} {X'0005616261636B03010200030266740302020003046E646F6E03030200'}

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.2 {
  UPDATE ft_segdir SET root = blob(
    '0005616261636B03010200 FFFFFFFF0702 66740302020003046E646F6E03030200'
  );
}

do_catchsql_test 1.3 {

} {12 3}

do_execsql_test 2.2 {
  SELECT quote(block) FROM ft_segments WHERE blockid=2
} {X'00056162633130031F0200'}

db func blob blob
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 2.3.1 {
  UPDATE ft_segments SET block = 
    blob('00056162633130031F0200 FFFFFFFF07FF55 66740302020003046E646F6E03030200')
    WHERE blockid=2;
} {}
do_catchsql_test 2.3.2 {
  INSERT INTO ft(ft) VALUES('merge=1,4');

} {1 5}

do_execsql_test 3.1 {
  SELECT quote(root) FROM ft_segdir
} {X'0101056162633132040136030132030136'}

db func blob blob
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 3.2 {
  UPDATE ft_segdir 
  SET root = blob('0101056162633132FFFFFFFF070236030132030136');
}

do_catchsql_test 3.1 {
  SELECT * FROM ft WHERE ft MATCH 'abc20'
} {1 {database disk image is malformed}}

finish_test

  execsql {
    INSERT INTO t1(t1) VALUES('optimize');
    SELECT substr(hex(root), 1, 2) FROM t1_segdir;
  }
} {03}

# Test the "missing entry" case:
sqlite3_db_config db DEFENSIVE 0
do_test fts3cov-2.2 {
  set root [db one {SELECT root FROM t1_segdir}]
  read_fts3varint [string range $root 1 end] left_child
  execsql { DELETE FROM t1_segments WHERE blockid = $left_child }
} {}
do_error_test fts3cov-2.3 {
  SELECT * FROM t1 WHERE t1 MATCH 'c*'
} {SQL logic error}

# Test the "replaced with NULL" case:
do_test fts3cov-2.4 {
  execsql { INSERT INTO t1_segments VALUES($left_child, NULL) }
} {}
sqlite3_db_config db DEFENSIVE 1
do_error_test fts3cov-2.5 {
  SELECT * FROM t1 WHERE t1 MATCH 'cloud'
} {SQL logic error}

#--------------------------------------------------------------------------
# The following tests are to test the effects of OOM errors while storing
# terms in the pending-hash table. Specifically, while creating doclist

}
do_execsql_test 15.1 {
  SELECT rowid FROM t15 WHERE t15 MATCH '"abc* def2"'
} {1 2}

# Test a corruption case.
#
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 16.1 {
  CREATE VIRTUAL TABLE t16 USING fts4;
  INSERT INTO t16 VALUES('theoretical work to examine the relationship');
  INSERT INTO t16 VALUES('solution of our problems on the invisible');
  DELETE FROM t16_content WHERE rowid = 2;
}
do_catchsql_test 16.2 {

  execsql {
    SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
  }
} {{Index already optimal} 1 0}

# Even if we move things around, still does nothing.
sqlite3_db_config db DEFENSIVE 0
do_test fts3d-5.1 {
  execsql {
    UPDATE t1_segdir SET level = 2 WHERE level = 1 AND idx = 0;
    SELECT OPTIMIZE(t1) FROM t1 LIMIT 1;
    SELECT level, idx FROM t1_segdir ORDER BY level, idx;
  }
} {{Index already optimal} 2 0}
sqlite3_db_config db DEFENSIVE 1


# ALTER TABLE RENAME should work regardless of the database encoding.
#
do_test fts3d-6.0 {
  db close
  forcedelete test.db

  4  {SELECT snippet(t1) FROM t1 WHERE t1 MATCH '"a longer phrase"'}  
     {"an instance of <b>a</b> <b>longer</b> <b>phrase</b>"}
  5  {SELECT rowid FROM t1 WHERE t1 MATCH 'a dog'}                   {1}
}

do_select_tests 1.2 $tests

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.3 {
  SELECT count(*) FROM t1_segments WHERE length(block)>10000;
  UPDATE t1_segments 
    SET block = zeroblob(length(block)) 
    WHERE length(block)>10000;
} {1}
sqlite3_db_config db DEFENSIVE 1

do_select_tests 1.4 $tests

# Drop the table. It is corrupt now anyhow, so not useful for subsequent tests.
#
do_execsql_test 1.5 { DROP TABLE t1 }

    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
  }
  3 {
    set dmt_modes {0 1 2}
    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
    add_empty_records 1000
    sqlite3_db_config db DEFENSIVE 0
    execsql $zero_long_doclists
    sqlite3_db_config db DEFENSIVE 1
  }
  4 {
    set dmt_modes 0
    execsql { CREATE VIRTUAL TABLE t1 USING FTS4 }
    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
    add_empty_records 1000
    execsql "INSERT INTO t1(t1) VALUES('optimize')"
    sqlite3_db_config db DEFENSIVE 0
    execsql $zero_long_doclists
    sqlite3_db_config db DEFENSIVE 1
  }
  5 {
    set dmt_modes 0
    execsql { CREATE VIRTUAL TABLE t1 USING FTS4(matchinfo=fts3) }
    foreach doc $data { execsql { INSERT INTO t1 VALUES($doc) } }
    add_empty_records 1000
    sqlite3_db_config db DEFENSIVE 0
    execsql $zero_long_doclists
    sqlite3_db_config db DEFENSIVE 1
  }
} {

  execsql { DROP TABLE IF EXISTS t1 }
  eval $setup
  set ::testprefix fts3defer-2.$tn
  set DO_MALLOC_TEST 0

  INSERT INTO t1 VALUES('');
  INSERT INTO t1 VALUES('');
  INSERT INTO t1 VALUES('');
  INSERT INTO t1 VALUES('');
  INSERT INTO t1 VALUES('');
  INSERT INTO t1(t1) VALUES('optimize');
}
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.1.4 {
  SELECT count(*) FROM t1_segments WHERE length(block)>10000;
  UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000;
} {2}
sqlite3_db_config db DEFENSIVE 1

do_execsql_test 1.2.0 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';

foreach {tn sql} {
  1 {}
  2 { INSERT INTO t2(t2) VALUES('optimize') }
  3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  sqlite3_db_config db DEFENSIVE 0
  execsql $sql
  sqlite3_db_config db DEFENSIVE 1

  do_execsql_test 2.2.$tn.1 {
    SELECT mit(matchinfo(t2, 'pcxnal')) FROM t2 WHERE t2 MATCH 'a b';
  } [list                                          \
    [list 2 1  1 54 54  1 3 3  54 372 8]        \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
  ]

foreach {tn sql} {
  1 {}
  2 { INSERT INTO t3(t3) VALUES('optimize') }
  3 { UPDATE t3_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  sqlite3_db_config db DEFENSIVE 0
  execsql $sql
  sqlite3_db_config db DEFENSIVE 1
  do_execsql_test 2.4.$tn {
    SELECT docid, mit(matchinfo(t3, 'pcxnal')) FROM t3 WHERE t3 MATCH '"a b c"';
  } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}}
}


finish_test

do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }

ifcapable fts4_deferred {
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test 4.4.0.2 {
    UPDATE t5_segments 
    SET block = zeroblob(length(block)) 
    WHERE length(block)>10000;
  }
  sqlite3_db_config db DEFENSIVE 1
}

do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }

  CREATE VIRTUAL TABLE t9 USING fts4;
  INSERT INTO t9 VALUES(
    'this record is used to try to dectect corruption'
  );
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {{0 0 20 2 0 0 27 2}}

sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 6.2 {
  UPDATE t9_content SET c0content = 'this record is used to'; 
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {1 {database disk image is malformed}}
sqlite3_db_config db DEFENSIVE 1

#-------------------------------------------------------------------------
# Test the outcome of matchinfo() when used within a query that does not
# use the full-text index (i.e. lookup by rowid or full-table scan).
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t10 USING fts4;

  execsql { INSERT INTO t11(t11) VALUES('optimize') }
} {}
do_execsql_test 8.3 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {{204 1 3 3 0} {204 1 3 3 0} {204 1 3 3 0}}

# Corruption related tests.
sqlite3_db_config db DEFENSIVE 0
do_execsql_test  8.4.1.1 { UPDATE t11_stat SET value = X'0000'; }
do_catchsql_test 8.5.1.2 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {1 {database disk image is malformed}}

do_execsql_test  8.4.2.1 { UPDATE t11_stat SET value = X'00'; }
do_catchsql_test 8.5.2.2 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {1 {database disk image is malformed}}

do_execsql_test  8.4.3.1 { UPDATE t11_stat SET value = NULL; }
do_catchsql_test 8.5.3.2 {
  SELECT mit(matchinfo(t11, 'nxa')) FROM t11 WHERE t11 MATCH 'a*'
} {1 {database disk image is malformed}}
sqlite3_db_config db DEFENSIVE 1

#-------------------------------------------------------------------------
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t12 USING fts4;
  INSERT INTO t12 VALUES('a b c d');
  SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'a NEAR/1 d OR a';
} {{0 0 0 0 0 0 1 1 1}}

  }
  do_execsql_test 4.1 {
    SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
  } {8000}
  do_execsql_test 4.2 {
    SELECT quote(value) from t4_stat where id=0
  } {X'C03EC0B204C0A608'}
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test 4.3 {
    UPDATE t4_stat SET value = X'C03EC0B204C0A60800' WHERE id=0;
  }
  do_catchsql_test 4.4 {
    SELECT count(*) FROM t4 WHERE t4 MATCH '"a b c" OR "c a b"'
  } {1 {database disk image is malformed}}
  do_execsql_test 4.5 {

  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
sqlite3_db_config db DEFENSIVE 0
do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' }
sqlite3_db_config db DEFENSIVE 1
do_select_tests 5.4 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize

  do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10]

  set off [string first "onehundred " $numbers]
  do_offsets_test $T.2.2 {onehundred} \
    [list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10]

  # Test a corruption case:
  sqlite3_db_config db DEFENSIVE 0
  execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers }
  sqlite3_db_config db DEFENSIVE 1
  do_error_test $T.2.3 {
    SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred'
  } {database disk image is malformed}
  
  ##########################################################################
  # Test the snippet function.
  #

  }
  3 {
    DELETE FROM t1_segdir WHERE level=0 AND idx=(
      SELECT max(idx) FROM t1_segdir WHERE level=0
    );
  }
} {
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test  1.2.1.$tn "BEGIN; $disruption"
  sqlite3_db_config db DEFENSIVE 1
  do_catchsql_test 1.2.2.$tn {
    INSERT INTO t1 (t1) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  1.2.3.$tn "ROLLBACK"
}

do_test 1.3 { fts_integrity db t1 } {ok}

  }
  3 {
    DELETE FROM t2_segdir WHERE level=0 AND idx=(
      SELECT max(idx) FROM t2_segdir WHERE level=1024
    );
  }
} {
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test  2.2.1.$tn "BEGIN; $disruption"
  sqlite3_db_config db DEFENSIVE 1
  do_catchsql_test 2.2.2.$tn {
    INSERT INTO t2 (t2) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  2.2.3.$tn "ROLLBACK"
}

  }
  2 {
    UPDATE t3_content SET langid=langid+1 WHERE rowid = (
      SELECT max(rowid) FROM t3_content
    )
  }
} {
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test  3.2.1.$tn "BEGIN; $disruption"
  sqlite3_db_config db DEFENSIVE 1
  do_catchsql_test 3.2.2.$tn {
    INSERT INTO t3 (t3) VALUES('integrity-check')
  } {1 {database disk image is malformed}}
  do_execsql_test  3.2.3.$tn "ROLLBACK"
}

#--------------------------------------------------------------------------
# Test case 4.*
#
# Test that the integrity-check works if there are "notindexed" columns.
#
do_execsql_test 4.0 {
  CREATE VIRTUAL TABLE t4 USING fts4(a, b, c, notindexed=b);
  INSERT INTO t4 VALUES('text one', 'text two', 'text three');
  INSERT INTO t4(t4) VALUES('integrity-check');
}

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 4.1 {
  PRAGMA writable_schema = 1;
  UPDATE sqlite_master 
    SET sql = 'CREATE VIRTUAL TABLE t4 USING fts4(a, b, c)' 
    WHERE name = 't4';
}

  INSERT INTO t5 VALUES('and the stockmen tell the story of his ride');
}

do_execsql_test 5.1 {
  INSERT INTO t5(t5) VALUES('integrity-check');
} {}

sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 5.2 {
  INSERT INTO t5_content VALUES(5, 'his hardy mountain pony');
  INSERT INTO t5(t5) VALUES('integrity-check');
} {1 {database disk image is malformed}}

do_execsql_test 5.3 ROLLBACK

do_execsql_test 5.4 {
  CREATE VIRTUAL TABLE t5 USING fts4(a, prefix="1,2,3");
  INSERT INTO t5(t5) VALUES('integrity-check');
} {}

finish_test

ifcapable !fts3 {
  finish_test
  return
}

source $testdir/genesis.tcl

sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; }

do_test 1.2 {
  foreach L {
    {"See here, young man," said Mulga Bill, "from Walgett to the sea,}
    {From Conroy's Gap to Castlereagh, there's none can ride like me.}
    {I'm good all round at everything as everybody knows,}

    } $expect
  }
  
  do_execsql_test 4.4.1 {
    SELECT quote(value) FROM t4_stat WHERE rowid=1
  } {X'0006'}
  
  sqlite3_db_config db DEFENSIVE 0
  do_execsql_test 4.4.2 {
    DELETE FROM t4_stat WHERE rowid=1;
    INSERT INTO t4(t4) VALUES('merge=1,12');
    SELECT level, group_concat(idx, ' ') FROM t4_segdir GROUP BY level;
  } "0 {0 1 2 3 4 5}                     1 0"
  sqlite3_db_config db DEFENSIVE 1
  
  
  #-------------------------------------------------------------------------
  # Test cases 5.*
  #
  # Test that if a crisis-merge occurs that disrupts an ongoing incremental
  # merge, the next call to "merge=A,B" identifies this and starts a new

# Argument $db is an open database handle. $tbl is the name of an FTS3/4
# table with the database. This command rearranges the contents of the
# %_segdir table so that all segments within each index are on the same
# level. This means that the 'merge' command can then be used for an
# incremental optimize routine.
#
proc prepare_for_optimize {db tbl} {
  sqlite3_db_config $db DEFENSIVE 0
  $db eval [string map [list % $tbl] {
    BEGIN;
      CREATE TEMP TABLE tmp_segdir(
        level, idx, start_block, leaves_end_block, end_block, root
      );

      INSERT INTO temp.tmp_segdir 

  
      DELETE FROM %_segdir;
      INSERT INTO %_segdir SELECT * FROM temp.tmp_segdir;
      DROP TABLE temp.tmp_segdir;
  
    COMMIT;
  }]
  sqlite3_db_config $db DEFENSIVE 1
}

do_test 1.1 {
  execsql { CREATE VIRTUAL TABLE t2 USING fts4(words, prefix="1,2,3") }
  foreach {docid words} [db eval { SELECT * FROM t1 }] {
    execsql { INSERT INTO t2(docid, words) VALUES($docid, $words) }
  }

      if {[info exists ::G(perm:presql)]} {
        [lindex $args 0] eval $::G(perm:presql)
      }
      if {[info exists ::G(perm:dbconfig)]} {
        set ::dbhandle [lindex $args 0]
        uplevel #0 $::G(perm:dbconfig)
      }
      sqlite3_db_config [lindex $args 0] DEFENSIVE 1
      set res
    } else {
      # This command is not opening a new database connection. Pass the
      # arguments through to the C implementation as the are.
      #
      uplevel 1 sqlite_orig $args
    }

  }
}

proc getFileRetries {} {
  if {![info exists ::G(file-retries)]} {
    #
    # NOTE: Return the default number of retries for [file] operations.  A