# 2014 May 12
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS4 module.
#
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts4growth
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
finish_test
return
}
source $testdir/genesis.tcl
do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; }
do_test 1.2 {
foreach L {
{"See here, young man," said Mulga Bill, "from Walgett to the sea,}
{From Conroy's Gap to Castlereagh, there's none can ride like me.}
{I'm good all round at everything as everybody knows,}
{Although I'm not the one to talk -- I hate a man that blows.}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql { SELECT end_block, length(root) FROM x1_segdir }
} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115}
do_execsql_test 1.3 {
INSERT INTO x1(x1) VALUES('optimize');
SELECT level, end_block, length(root) FROM x1_segdir;
} {0 {0 394} 394}
do_test 1.4 {
foreach L {
{But riding is my special gift, my chiefest, sole delight;}
{Just ask a wild duck can it swim, a wildcat can it fight.}
{There's nothing clothed in hair or hide, or built of flesh or steel,}
{There's nothing walks or jumps, or runs, on axle, hoof, or wheel,}
{But what I'll sit, while hide will hold and girths and straps are tight:}
{I'll ride this here two-wheeled concern right straight away at sight."}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql {
INSERT INTO x1(x1) VALUES('merge=4,4');
SELECT level, end_block, length(root) FROM x1_segdir;
}
} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2}
do_execsql_test 1.5 {
SELECT length(block) FROM x1_segments;
} {658 {}}
do_test 1.6 {
foreach L {
{'Twas Mulga Bill, from Eaglehawk, that sought his own abode,}
{That perched above Dead Man's Creek, beside the mountain road.}
{He turned the cycle down the hill and mounted for the fray,}
{But 'ere he'd gone a dozen yards it bolted clean away.}
{It left the track, and through the trees, just like a silver steak,}
{It whistled down the awful slope towards the Dead Man's Creek.}
{It shaved a stump by half an inch, it dodged a big white-box:}
{The very wallaroos in fright went scrambling up the rocks,}
{The wombats hiding in their caves dug deeper underground,}
{As Mulga Bill, as white as chalk, sat tight to every bound.}
{It struck a stone and gave a spring that cleared a fallen tree,}
{It raced beside a precipice as close as close could be;}
{And then as Mulga Bill let out one last despairing shriek}
{It made a leap of twenty feet into the Dead Man's Creek.}
} {
execsql { INSERT INTO x1 VALUES($L) }
}
execsql {
SELECT level, end_block, length(root) FROM x1_segdir;
}
} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117}
do_execsql_test 1.7 {
SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130);
} {1377}
#-------------------------------------------------------------------------
#
do_execsql_test 2.1 {
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x2 USING fts4;
}
fts_kjv_genesis
do_test 2.2 {
foreach id [db eval {SELECT docid FROM t1}] {
execsql {
INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id
}
}
foreach id [db eval {SELECT docid FROM t1}] {
execsql {
INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id
}
if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break
}
} {}
do_execsql_test 2.3 {
SELECT count(*) FROM x2_segdir WHERE level=2;
SELECT count(*) FROM x2_segdir WHERE level=3;
} {6 0}
do_execsql_test 2.4 {
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT count(*) FROM x2_segdir WHERE level=2;
SELECT count(*) FROM x2_segdir WHERE level=3;
} {6 1}
do_execsql_test 2.5 {
SELECT end_block FROM x2_segdir WHERE level=3;
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT end_block FROM x2_segdir WHERE level=3;
INSERT INTO x2(x2) VALUES('merge=4,4');
SELECT end_block FROM x2_segdir WHERE level=3;
} {{3828 3430} {3828 10191} {3828 14109}}
do_execsql_test 2.6 {
SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE
blockid BETWEEN start_block AND leaves_end_block
AND level=3
} {14109}
do_execsql_test 2.7 {
INSERT INTO x2(x2) VALUES('merge=1000,4');
SELECT end_block FROM x2_segdir WHERE level=3;
} {{3828 86120}}
do_execsql_test 2.8 {
SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE
blockid BETWEEN start_block AND leaves_end_block
AND level=3
} {86120}
#--------------------------------------------------------------------------
# Test that delete markers are removed from FTS segments when possible.
# It is only possible to remove delete markers when the output of the
# merge operation will become the oldest segment in the index.
#
# 3.1 - when the oldest segment is created by an 'optimize'.
# 3.2 - when the oldest segment is created by an incremental merge.
# 3.3 - by a crisis merge.
#
proc insert_doc {args} {
foreach iDoc $args {
set L [lindex {
{In your eagerness to engage the Trojans,}
{don’t any of you charge ahead of others,}
{trusting in your strength and horsemanship.}
{And don’t lag behind. That will hurt our charge.}
{Any man whose chariot confronts an enemy’s}
{should thrust with his spear at him from there.}
{That’s the most effective tactic, the way}
{men wiped out city strongholds long ago —}
{their chests full of that style and spirit.}
} [expr $iDoc%9]]
execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) }
}
}
proc delete_doc {args} {
foreach iDoc $args {
execsql { DELETE FROM x3 WHERE docid = $iDoc }
}
}
proc second {x} { lindex $x 1 }
db func second second
do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 }
do_test 3.1.1 {
db transaction { insert_doc 1 2 3 4 5 6 }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 412}
do_test 3.1.2 {
delete_doc 1 2 3 4 5 6
execsql { SELECT count(*) FROM x3_segdir }
} {0}
do_test 3.1.3 {
db transaction {
insert_doc 1 2 3 4 5 6 7 8 9
delete_doc 9 8 7
}
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 591 0 1 65 0 2 72 0 3 76}
do_test 3.1.4 {
execsql { INSERT INTO x3(x3) VALUES('optimize') }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {0 0 412}
do_test 3.2.1 {
execsql { DELETE FROM x3 }
insert_doc 8 7 6 5 4 3 2 1
delete_doc 7 8
execsql { SELECT count(*) FROM x3_segdir }
} {10}
do_test 3.2.2 {
execsql { INSERT INTO x3(x3) VALUES('merge=500,10') }
execsql { SELECT level, idx, second(end_block) FROM x3_segdir }
} {1 0 412}
# This assumes the crisis merge happens when there are already 16
# segments and one more is added.
#
do_test 3.3.1 {
execsql { DELETE FROM x3 }
insert_doc 1 2 3 4 5 6 7 8 9 10 11
delete_doc 11 10 9 8 7
execsql { SELECT count(*) FROM x3_segdir }
} {16}
do_test 3.3.2 {
insert_doc 12
execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 }
} {1 0 412}
#--------------------------------------------------------------------------
do_execsql_test 4.1 {
DROP TABLE IF EXISTS x4;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x4 USING fts4(words);
}
do_test 4.2 {
fts_kjv_genesis
execsql { INSERT INTO x4 SELECT words FROM t1 }
execsql { INSERT INTO x4 SELECT words FROM t1 }
} {}
do_execsql_test 4.3 {
SELECT level, idx, second(end_block) FROM x4_segdir
} {0 0 117483 0 1 118006}
do_execsql_test 4.4 {
INSERT INTO x4(x4) VALUES('merge=10,2');
SELECT count(*) FROM x4_segdir;
} {3}
breakpoint
do_execsql_test 4.5 {
INSERT INTO x4(x4) VALUES('merge=10,2');
SELECT count(*) FROM x4_segdir;
} {3}
if 0 {
do_execsql_test 3.1 {
DROP TABLE IF EXISTS x2;
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(docid, words);
CREATE VIRTUAL TABLE x2 USING fts4;
}
fts_kjv_genesis
proc t1_to_x2 {} {
foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] {
execsql {
DELETE FROM x2 WHERE docid=$id;
INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id;
}
}
}
#do_test 3.2 {
#t1_to_x2
#execsql {SELECT level, count(*) FROM x2_segdir GROUP BY level}
#} {0 13 1 15 2 5}
proc second {x} { lindex $x 1 }
db func second second
for {set i 0} {$i <1000} {incr i} {
t1_to_x2
db eval {
SELECT level, group_concat( second(end_block), ' ' ) AS c FROM x2_segdir GROUP BY level;
} {
puts "$i.$level: $c"
}
}
}
finish_test