SQLite

Check-in [903ec5126d]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add fts4merge3.test, for testing that older versions of FTS4 may interoperate with incr-merge capable versions.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts4-incr-merge
Files: files | file ages | folders
SHA1: 903ec5126dd981da6d7bab45c568f34b99446159
User & Date: dan 2012-03-21 14:34:23.781
Context
2012-03-22
16:48
Following an incr-merge operation that does not completely consume its input segments, store context in the rowid==1 row of the %_stat table that allows the next incr-merge to pick up where the previous left off. (check-in: ab0a4f44fb user: dan tags: fts4-incr-merge)
2012-03-21
14:34
Add fts4merge3.test, for testing that older versions of FTS4 may interoperate with incr-merge capable versions. (check-in: 903ec5126d user: dan tags: fts4-incr-merge)
2012-03-20
17:04
Merge trunk changes into the fts4-incr-merge branch. (check-in: f61d5fb028 user: drh tags: fts4-incr-merge)
Changes
Unified Diff Ignore Whitespace Patch
Changes to test/backcompat.test.
23
24
25
26
27
28
29

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
# for documentation of the available commands.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl

db close

# Search for binaries to test against. Any executable files that match
# our naming convention are assumed to be testfixture binaries to test
# against.
#
set binaries [list]
set pattern "[file tail [info nameofexec]]?*"
if {$tcl_platform(platform)=="windows"} {
  set pattern [string map {\.exe {}} $pattern]
}
foreach file [glob -nocomplain $pattern] {
  if {[file executable $file] && [file isfile $file]} {lappend binaries $file}
}
if {[llength $binaries]==0} {
  puts "WARNING: No historical binaries to test against."
  puts "WARNING: No backwards-compatibility tests have been run."
  finish_test
  return
}
proc get_version {binary} {
  set chan [launch_testfixture $binary]
  set v [testfixture $chan { sqlite3 -version }]
  close $chan
  set v
}
foreach bin $binaries {
  puts -nonewline "Testing against $bin - "
  flush stdout
  puts "version [get_version $bin]"
}

proc do_backcompat_test {rv bin1 bin2 script} {

  forcedelete test.db

  if {$bin1 != ""} { set ::bc_chan1 [launch_testfixture $bin1] }
  set ::bc_chan2 [launch_testfixture $bin2]







>


<
<
<
<
|
<
<
<
<
<
<
<
<
<
<



<
<
<
<
<
<
<
<
<
<
<







23
24
25
26
27
28
29
30
31
32




33










34
35
36











37
38
39
40
41
42
43
# for documentation of the available commands.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/lock_common.tcl
source $testdir/malloc_common.tcl
source $testdir/bc_common.tcl
db close





if {"" == [bc_find_binaries backcompat.test]} {










  finish_test
  return
}












proc do_backcompat_test {rv bin1 bin2 script} {

  forcedelete test.db

  if {$bin1 != ""} { set ::bc_chan1 [launch_testfixture $bin1] }
  set ::bc_chan2 [launch_testfixture $bin2]
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  catch { close $::bc_chan2 }
  catch { close $::bc_chan1 }
}

array set ::incompatible [list]
proc do_allbackcompat_test {script} {

  foreach bin $::binaries {
    set nErr [set_test_counter errors]
    foreach dir {0 1} {

      set bintag [string map {testfixture {}} $bin]
      set bintag [string map {\.exe {}} $bintag]
      if {$bintag == ""} {set bintag self}
      set ::bcname ".$bintag.$dir."







|







65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
  catch { close $::bc_chan2 }
  catch { close $::bc_chan1 }
}

array set ::incompatible [list]
proc do_allbackcompat_test {script} {

  foreach bin $::BC(binaries) {
    set nErr [set_test_counter errors]
    foreach dir {0 1} {

      set bintag [string map {testfixture {}} $bin]
      set bintag [string map {\.exe {}} $bintag]
      if {$bintag == ""} {set bintag self}
      set ::bcname ".$bintag.$dir."
Added test/bc_common.tcl.
























































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76



proc bc_find_binaries {zCaption} {
  # Search for binaries to test against. Any executable files that match
  # our naming convention are assumed to be testfixture binaries to test
  # against.
  #
  set binaries [list]
  set pattern "[file tail [info nameofexec]]?*"
  if {$::tcl_platform(platform)=="windows"} {
    set pattern [string map {\.exe {}} $pattern]
  }
  foreach file [glob -nocomplain $pattern] {
    if {[file executable $file] && [file isfile $file]} {lappend binaries $file}
  }

  if {[llength $binaries]==0} {
    puts "WARNING: No historical binaries to test against."
    puts "WARNING: Omitting backwards-compatibility tests $zFile"
  }

  foreach bin $binaries {
    puts -nonewline "Testing against $bin - "
    flush stdout
    puts "version [get_version $bin]"
  }

  set ::BC(binaries) $binaries
  return $binaries
}

proc get_version {binary} {
  set chan [launch_testfixture $binary]
  set v [testfixture $chan { sqlite3 -version }]
  close $chan
  set v
}

proc do_bc_test {bin script} {

  forcedelete test.db
  set ::bc_chan [launch_testfixture $bin]

  proc code1 {tcl} { uplevel #0 $tcl }
  proc code2 {tcl} { testfixture $::bc_chan $tcl }
  proc sql1 sql { code1 [list db eval $sql] }
  proc sql2 sql { code2 [list db eval $sql] }

  code1 { sqlite3 db test.db }
  code2 { sqlite3 db test.db }

  set bintag [string map {testfixture {}} $bin]
  set bintag [string map {\.exe {}} $bintag]
  if {$bintag == ""} {set bintag self}
  set saved_prefix $::testprefix
  append ::testprefix ".$bintag"

  uplevel $script

  set ::testprefix $saved_prefix

  catch { code1 { db close } }
  catch { code2 { db close } }
  catch { close $::bc_chan }
}

proc do_all_bc_test {script} {
  foreach bin $::BC(binaries) {
    uplevel [list do_bc_test $bin $script]
  }
}




Changes to test/fts3_common.tcl.
10
11
12
13
14
15
16





























17
18
19
20
21
22
23
#***********************************************************************
#
# This file contains common code used the fts3 tests. At one point
# equivalent functionality was implemented in C code. But it is easier
# to use Tcl.
#































#-------------------------------------------------------------------------
# USAGE: fts3_build_db_1 N
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t1".
#







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
#***********************************************************************
#
# This file contains common code used the fts3 tests. At one point
# equivalent functionality was implemented in C code. But it is easier
# to use Tcl.
#

#-------------------------------------------------------------------------
# INSTRUCTIONS
#
# The following commands are available:
#
#   fts3_build_db_1 N
#     Using database handle [db] create an FTS4 table named t1 and populate
#     it with N rows of data. N must be less than 10,000. Refer to the
#     header comments above the proc implementation below for details.
#
#   fts3_build_db_2 N
#     Using database handle [db] create an FTS4 table named t2 and populate
#     it with N rows of data. N must be less than 100,000. Refer to the
#     header comments above the proc implementation below for details.
#
#   fts3_integrity_check TBL
#     TBL must be an FTS table in the database currently opened by handle
#     [db]. This proc loads and tokenizes all documents within the table,
#     then checks that the current contents of the FTS index matches the
#     results.
#
#   fts3_terms TBL WHERE
#     Todo.
#
#   fts3_doclist TBL TERM WHERE
#     Todo.
#
#
#

#-------------------------------------------------------------------------
# USAGE: fts3_build_db_1 N
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t1".
#
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63



64
65
66
67
68
69
70
    lappend y [lindex $ywords [expr ($i / 1)   % 10]]

    db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) }
  }
}

#-------------------------------------------------------------------------
# USAGE: fts3_build_db_2 N
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t2".
#
proc fts3_build_db_2 {n} {

  if {$n > 100000} {error "n must be <= 100000"}

  db eval { CREATE VIRTUAL TABLE t2 USING fts4 }




  set chars [list a b c d e f g h  i j k l m n o p  q r s t u v w x  y z ""]

  for {set i 0} {$i < $n} {incr i} {
    set word ""
    set nChar [llength $chars]
    append word [lindex $chars [expr {($i / 1)   % $nChar}]]







|




|



|
>
>
>







76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
    lappend y [lindex $ywords [expr ($i / 1)   % 10]]

    db eval { INSERT INTO t1(docid, x, y) VALUES($i, $x, $y) }
  }
}

#-------------------------------------------------------------------------
# USAGE: fts3_build_db_2 N ARGS
#
# Build a sample FTS table in the database opened by database connection 
# [db]. The name of the new table is "t2".
#
proc fts3_build_db_2 {n args} {

  if {$n > 100000} {error "n must be <= 100000"}

  set sql "CREATE VIRTUAL TABLE t2 USING fts4(content"
  foreach a $args { append sql ", " $a }
  append sql ")"
  db eval $sql

  set chars [list a b c d e f g h  i j k l m n o p  q r s t u v w x  y z ""]

  for {set i 0} {$i < $n} {incr i} {
    set word ""
    set nChar [llength $chars]
    append word [lindex $chars [expr {($i / 1)   % $nChar}]]
Added test/fts4merge3.test.
































































































































































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# 2012 March 06
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the incremental merge function.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
source $testdir/fts3_common.tcl
source $testdir/lock_common.tcl
source $testdir/bc_common.tcl

set ::testprefix fts4merge3

if {"" == [bc_find_binaries backcompat.test]} {
  finish_test
  return
}

do_all_bc_test {

  sql2 { PRAGMA page_size = 512 }
  if { 0==[catch { sql2 { CREATE VIRTUAL TABLE x USING fts4 } } ] } {

    # Build a large database.
    set msg "this takes around 12 seconds"
    do_test "1.1 ($msg)" { fts3_build_db_2 20000 } {}

    # Run some queries on it, using the old and new versions.
    do_test 1.2 { sql1 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485}
    do_test 1.3 { sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" } {1485}

    do_test 1.4 { sql2 "PRAGMA page_count" } {1286}
    do_test 1.5 { sql2 { 
      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
    } } [list 0 15    1 1     2 14    3 4]

    # Run some incr-merge operations on the db.
    for {set i 0} {$i<10} {incr i} {
      do_test 1.6.$i.1 { sql1 { INSERT INTO t2(t2) VALUES('merge=2,2') } } {}
      do_test 1.6.$i.2 { 
        sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
      } {1485}
    }

    do_test 1.7 { sql2 { 
      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
    } } [list 0 15 1 1 2 14 3 4 4 1]

    # Using the old connection, insert many rows. 
    do_test 1.8 {
      for {set i 0} {$i < 1500} {incr i} {
        sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i"
      }
    } {}

    do_test 1.9 { sql2 { 
      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
    } } {0 11 1 15 2 3 3 5 4 1}

    # Run a big incr-merge operation on the db.
    do_test 1.10 { sql1 { INSERT INTO t2(t2) VALUES('merge=2000,2') } } {}
    do_test 1.11 { 
      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
    } {1485 21485}

    do_test 1.12 {
      for {set i 0} {$i < 1500} {incr i} {
        sql2 "INSERT INTO t2 SELECT content FROM t2 WHERE docid = $i"
      }
    } {}
    do_test 1.13 { 
      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
    } {1485 21485 22985}

    do_test 1.14 { 
      sql2 "INSERT INTO t2(t2) VALUES('optimize')"
      sql2 "SELECT docid FROM t2 WHERE t2 MATCH 'abc'" 
    } {1485 21485 22985}

    do_test 1.15 { sql2 { 
      SELECT level, count(*) FROM t2_segdir GROUP BY level ORDER BY 1
    } } {5 1}
  }
}


finish_test