# 2015 Apr 24
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file tests that FTS5 handles corrupt databases (i.e. internal
# inconsistencies in the backing tables) correctly. In this case
# "correctly" means without crashing.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5corrupt3
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_may_be_corrupt 1
proc create_t1 {} {
expr srand(0)
db func rnddoc fts5_rnddoc
db eval {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
INSERT INTO t1 SELECT rnddoc(10) FROM ii;
}
}
if 1 {
# Create a simple FTS5 table containing 100 documents. Each document
# contains 10 terms, each of which start with the character "x".
#
do_test 1.0 { create_t1 } {}
do_test 1.1 {
# Pick out the rowid of the right-most b-tree leaf in the new segment.
set rowid [db one {
SELECT max(rowid) FROM t1_data WHERE ((rowid>>31) & 0x0F)==1
}]
set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
set {} {}
} {}
for {set i 0} {$i < $L} {incr i} {
do_test 1.2.$i {
catchsql {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
INSERT INTO t1(t1) VALUES('integrity-check');
}
} {1 {database disk image is malformed}}
catchsql ROLLBACK
}
#-------------------------------------------------------------------------
# Test that trailing bytes appended to the averages record are ignored.
#
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x);
INSERT INTO t2 VALUES(rnddoc(10));
INSERT INTO t2 VALUES(rnddoc(10));
SELECT length(block) FROM t2_data WHERE id=1;
} {2}
do_execsql_test 2.2 {
UPDATE t2_data SET block = block || 'abcd' WHERE id=1;
SELECT length(block) FROM t2_data WHERE id=1;
} {6}
do_execsql_test 2.2 {
INSERT INTO t2 VALUES(rnddoc(10));
SELECT length(block) FROM t2_data WHERE id=1;
} {2}
#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}
do_execsql_test 3.1 {
SELECT count(*) FROM t1_data;
} {105}
proc do_3_test {tn} {
set i 0
foreach ::rowid [db eval "SELECT rowid FROM t1_data WHERE rowid>100"] {
incr i
do_test $tn.$i {
db eval BEGIN
db eval {DELETE FROM t1_data WHERE rowid = $::rowid}
list [
catch { db eval {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'} } msg
] $msg
} {1 {database disk image is malformed}}
catch { db eval ROLLBACK }
}
}
do_3_test 3.2
do_execsql_test 3.3 {
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
} {}
do_3_test 3.4
do_test 3.5 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
}
for {set i 0} {$i < 1000} {incr i} {
set rnd [expr int(rand() * 1000)]
set doc [string repeat "x$rnd " [expr int(rand() * 3) + 1]]
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.6
do_test 3.7 {
execsql {
INSERT INTO t1(t1, rank) VALUES('pgsz', 40);
INSERT INTO t1 SELECT x FROM t1;
INSERT INTO t1(t1) VALUES('optimize');
}
} {}
do_3_test 3.8
do_test 3.9 {
execsql {
DELETE FROM t1;
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
} {}
do_3_test 3.10
#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
do_test 4.0 {
execsql {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
for {set i 0} {$i < 100} {incr i} {
set rnd [expr int(rand() * 100)]
set doc "x[string repeat $rnd 20]"
execsql { INSERT INTO t1(rowid, x) VALUES($i, $doc) }
}
execsql { INSERT INTO t1(t1) VALUES('optimize') }
} {}
set nErr 0
for {set i 1} {1} {incr i} {
set struct [db one {SELECT block FROM t1_data WHERE id=10}]
binary scan $struct c* var
set end [lindex $var end]
if {$end<=$i} break
lset var end [expr $end - $i]
set struct [binary format c* $var]
db close
sqlite3 db test.db
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;
}
do_test 4.1.$i {
incr nErr [catch { db eval { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } }]
set {} {}
} {}
catch { db eval ROLLBACK }
}
do_test 4.1.x { expr $nErr>45 } 1
#-------------------------------------------------------------------------
#
# The first argument passed to this command must be a binary blob
# containing an FTS5 leaf page. This command returns a copy of this
# blob, with the pgidx of the leaf page replaced by a single varint
# containing value $iVal.
#
proc rewrite_pgidx {blob iVal} {
binary scan $blob SS off1 szLeaf
if {$iVal<0 || $iVal>=128} {
error "$iVal out of range!"
} else {
set pgidx [binary format c $iVal]
}
binary format a${szLeaf}a* $blob $pgidx
}
reset_db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
INSERT INTO x1(x1, rank) VALUES('pgsz', 40);
BEGIN;
INSERT INTO x1 VALUES('xaaa xabb xccc xcdd xeee xeff xggg xghh xiii xijj');
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1 SELECT x FROM x1;
INSERT INTO x1(x1) VALUES('optimize');
COMMIT;
}
#db eval { SELECT fts5_decode(id, block) b from x1_data } { puts $b }
#
db func rewrite_pgidx rewrite_pgidx
set i 0
foreach rowid [db eval {SELECT rowid FROM x1_data WHERE rowid>100}] {
foreach val {2 100} {
do_test 5.2.$val.[incr i] {
catchsql {
BEGIN;
UPDATE x1_data SET block=rewrite_pgidx(block, $val) WHERE id=$rowid;
SELECT rowid FROM x1 WHERE x1 MATCH 'xa*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xb*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xc*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xd*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xe*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xf*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xg*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xh*';
SELECT rowid FROM x1 WHERE x1 MATCH 'xi*';
}
set {} {}
} {}
catch { db eval ROLLBACK }
}
}
#------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('bbbbb ccccc');
SELECT quote(block) FROM t1_data WHERE rowid>100;
} {X'000000180630626262626201020201056363636363010203040A'}
do_execsql_test 6.1.1 {
UPDATE t1_data SET block =
X'000000180630626262626201020201056161616161010203040A'
WHERE rowid>100;
}
do_catchsql_test 6.1.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
do_execsql_test 6.2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('aa bb cc dd ee');
SELECT pgno, quote(term) FROM t1_idx;
} {2 X'' 4 X'3064'}
do_execsql_test 6.2.1 {
UPDATE t1_idx SET term = X'3065' WHERE pgno=4;
}
do_catchsql_test 6.2.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
do_execsql_test 6.3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('abc abcdef abcdefghi');
SELECT quote(block) FROM t1_data WHERE id>100;
} {X'0000001C043061626301020204036465660102030703676869010204040808'}
do_execsql_test 6.3.1 {
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102035003676869010204040808'
------------------------------------------^^---------------------
WHERE id>100;
}
do_catchsql_test 6.3.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.3 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102030750676869010204040808'
--------------------------------------------^^-------------------
WHERE id>100;
}
do_catchsql_test 6.3.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.4 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C043061626301020204036465660102030707676869010204040850'
--------------------------------------------------------------^^-
WHERE id>100;
}
do_catchsql_test 6.3.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
do_execsql_test 6.3.6 {
ROLLBACK;
BEGIN;
UPDATE t1_data SET block =
X'0000001C503061626301020204036465660102030707676869010204040808'
----------^^-----------------------------------------------------
WHERE id>100;
}
do_catchsql_test 6.3.5 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#------------------------------------------------------------------------
#
reset_db
proc rnddoc {n} {
set map [list a b c d]
set doc [list]
for {set i 0} {$i < $n} {incr i} {
lappend doc "x[lindex $map [expr int(rand()*4)]]"
}
set doc
}
db func rnddoc rnddoc
do_test 7.0 {
execsql {
CREATE VIRTUAL TABLE t5 USING fts5(x);
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5(t5) VALUES('optimize');
}
} {}
do_test 7.1 {
foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
db eval BEGIN
db eval {DELETE FROM t5_data WHERE rowid = $i}
set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
if {$r != "1 {database disk image is malformed}"} { error $r }
db eval ROLLBACK
}
} {}
}
#------------------------------------------------------------------------
# Corruption within the structure record.
#
reset_db
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
INSERT INTO t1 VALUES('one', 'two');
}
do_test 9.1.1 {
set blob "12345678" ;# cookie
append blob "0105" ;# 1 level, total of 5 segments
append blob "06" ;# write counter
append blob "0002" ;# first level has 0 segments merging, 2 other.
append blob "450108" ;# first segment
execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
} {}
do_catchsql_test 9.1.2 {
SELECT * FROM t1('one AND two');
} {1 {database disk image is malformed}}
do_test 9.2.1 {
set blob "12345678" ;# cookie
append blob "0205" ;# 2 levels, total of 5 segments
append blob "06" ;# write counter
append blob "0001" ;# first level has 0 segments merging, 1 other.
append blob "450108" ;# first segment
execsql "REPLACE INTO t1_data VALUES(10, X'$blob')"
} {}
do_catchsql_test 9.2.2 {
SELECT * FROM t1('one AND two');
} {1 {database disk image is malformed}}
sqlite3_fts5_may_be_corrupt 0
finish_test