/ Check-in [f29c8fca]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a few extra coverage test cases for fts3.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-refactor
Files: files | file ages | folders
SHA1: f29c8fcade4aadeae3824975cf59f306c11c906b
User & Date: dan 2009-11-16 16:36:23
Context
2009-11-17
12:52
Improvements to the way fts3 reads the full-text index. check-in: 45c051e7 user: dan tags: fts3-refactor
2009-11-16
16:36
Add a few extra coverage test cases for fts3. check-in: f29c8fca user: dan tags: fts3-refactor
2009-11-15
06:50
Fixes to fts3 integrity check code. check-in: d3cae986 user: dan tags: fts3-refactor
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
...
490
491
492
493
494
495
496

497
498
499

500
501

502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
...
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
...
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
...
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
**     'xyz'   becomes   xyz
**     [pqr]   becomes   pqr
**     `mno`   becomes   mno
*/
void sqlite3Fts3Dequote(char *z){
  int quote;
  int i, j;
  if( z==0 ) return;
  quote = z[0];
  switch( quote ){
    case '\'':  break;
    case '"':   break;
    case '`':   break;                /* For MySQL compatibility */
    case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
    default:    return;
................................................................................
*/
static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
  Fts3Table *p = (Fts3Table *)pVtab;
  int i;

  assert( p->nPendingData==0 );


  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }


  if( p->pTokenizer ){

    p->pTokenizer->pModule->xDestroy(p->pTokenizer);
  }

  sqlite3_free(p->zSelectLeaves);
  sqlite3_free(p);
  return SQLITE_OK;
}

/*
** The xDisconnect() virtual table method.
*/
static int fts3DestroyMethod(sqlite3_vtab *pVtab){
  int rc;                         /* Return code */
  Fts3Table *p = (Fts3Table *)pVtab;

  /* Create a script to drop the underlying three storage tables. */
  char *zSql = sqlite3_mprintf(
................................................................................
  }

  /* Create the whole "CREATE TABLE" statement to pass to SQLite */
  zSql = sqlite3_mprintf(
      "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
  );

  if( !zSql || !zCols ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_declare_vtab(p->db, zSql);
  }

  sqlite3_free(zSql);
  sqlite3_free(zCols);
................................................................................
  char *zContentCols;             /* Columns of %_content table */
  char *zSql;                     /* SQL script to create required tables */

  /* Create a list of user columns for the content table */
  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
  for(i=0; zContentCols && i<p->nColumn; i++){
    char *z = p->azColumn[i];
    zContentCols = sqlite3_mprintf("%z, c%d%s", zContentCols, i+1, z);
  }

  /* Create the whole SQL script */
  zSql = sqlite3_mprintf(
      "CREATE TABLE %Q.'%q_content'(%s);"
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
      ");",
      p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
  );

  /* Unless a malloc() failure has occurred, execute the SQL script to 
  ** create the tables used to store data for this FTS3 virtual table.
  */
  if( zSql==0 || zContentCols==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
  }

  sqlite3_free(zSql);
  sqlite3_free(zContentCols);







|







 







>



>

<
>
|
|
<
<





|







 







|







 







|







 







|







431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
...
490
491
492
493
494
495
496
497
498
499
500
501
502

503
504
505


506
507
508
509
510
511
512
513
514
515
516
517
518
...
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
...
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
...
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
**     'xyz'   becomes   xyz
**     [pqr]   becomes   pqr
**     `mno`   becomes   mno
*/
void sqlite3Fts3Dequote(char *z){
  int quote;
  int i, j;

  quote = z[0];
  switch( quote ){
    case '\'':  break;
    case '"':   break;
    case '`':   break;                /* For MySQL compatibility */
    case '[':   quote = ']';  break;  /* For MS SqlServer compatibility */
    default:    return;
................................................................................
*/
static int fts3DisconnectMethod(sqlite3_vtab *pVtab){
  Fts3Table *p = (Fts3Table *)pVtab;
  int i;

  assert( p->nPendingData==0 );

  /* Free any prepared statements held */
  for(i=0; i<SizeofArray(p->aStmt); i++){
    sqlite3_finalize(p->aStmt[i]);
  }
  sqlite3_free(p->zSelectLeaves);


  /* Invoke the tokenizer destructor to free the tokenizer. */
  p->pTokenizer->pModule->xDestroy(p->pTokenizer);



  sqlite3_free(p);
  return SQLITE_OK;
}

/*
** The xDestroy() virtual table method.
*/
static int fts3DestroyMethod(sqlite3_vtab *pVtab){
  int rc;                         /* Return code */
  Fts3Table *p = (Fts3Table *)pVtab;

  /* Create a script to drop the underlying three storage tables. */
  char *zSql = sqlite3_mprintf(
................................................................................
  }

  /* Create the whole "CREATE TABLE" statement to pass to SQLite */
  zSql = sqlite3_mprintf(
      "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName
  );

  if( !zCols || !zSql ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_declare_vtab(p->db, zSql);
  }

  sqlite3_free(zSql);
  sqlite3_free(zCols);
................................................................................
  char *zContentCols;             /* Columns of %_content table */
  char *zSql;                     /* SQL script to create required tables */

  /* Create a list of user columns for the content table */
  zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY");
  for(i=0; zContentCols && i<p->nColumn; i++){
    char *z = p->azColumn[i];
    zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i+1, z);
  }

  /* Create the whole SQL script */
  zSql = sqlite3_mprintf(
      "CREATE TABLE %Q.'%q_content'(%s);"
      "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);"
      "CREATE TABLE %Q.'%q_segdir'("
................................................................................
      ");",
      p->zDb, p->zName, zContentCols, p->zDb, p->zName, p->zDb, p->zName
  );

  /* Unless a malloc() failure has occurred, execute the SQL script to 
  ** create the tables used to store data for this FTS3 virtual table.
  */
  if( zContentCols==0 || zSql==0 ){
    rc = SQLITE_NOMEM;
  }else{
    rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
  }

  sqlite3_free(zSql);
  sqlite3_free(zContentCols);

Changes to test/fts3malloc.test.

15
16
17
18
19
20
21
22
23
24
25
26
27

28
29
30
31
32
33

34
35



36
37
38
39
40
41
42
..
51
52
53
54
55
56
57
58
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
..
78
79
80
81
82
83
84








85
86
87
88
89
90
91
...
103
104
105
106
107
108
109


110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
...
148
149
150
151
152
153
154
155

156
157
158



























159
160
161
162
163
164
165

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
source $testdir/malloc_common.tcl
source $testdir/fts3_common.tcl

set sqlite_fts3_enable_parentheses 1

if 0 {
do_malloc_test fts3_malloc-1.1 -sqlbody {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
}

do_malloc_test fts3_malloc-1.2 -sqlprep {
  CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
} -sqlbody {
  DROP TABLE ft;
}
}


set DO_MALLOC_TEST 0




#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
# set to the expected output if the SELECT statement is successfully
# executed using [db eval].
................................................................................
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#
proc do_select_test {name sql result} {


  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} [list 0 $result]]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list [list 0 $result]]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
................................................................................
      }
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}









#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#
................................................................................
# transaction is rolled back, non-zero is returned.
#
# Parameter $name is the name to use for the test case (or test cases).
# The second parameter, $tbl, should be the name of the database table
# being modified. Parameter $sql contains the SQL statement to test.
#
proc do_write_test {name tbl sql} {


  # Figure out an statement to get a checksum for table $tbl.
  db eval "SELECT * FROM $tbl" V break
  set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl"

  # Calculate the initial table checksum.
  set cksum1 [db one $cksumsql]

  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} {0 {}}]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list {0 {}}]
    set modes [list 0 nofail]
................................................................................

proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
}

db close

file delete -force test.db test.db-journal
sqlite3 db test.db
sqlite3_db_config_lookaside db 0 0 0




























do_test fts3_malloc-2.0 {
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}







|
|
|
|
<
|
>
|
<
<
<
|
<
>
|
<
>
>
>







 







<

>


|
|


|







 







>
>
>
>
>
>
>
>







 







>
>







<







 







<
>
|
|
<
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







15
16
17
18
19
20
21
22
23
24
25

26
27
28



29

30
31

32
33
34
35
36
37
38
39
40
41
..
50
51
52
53
54
55
56

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
..
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
...
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

126
127
128
129
130
131
132
...
156
157
158
159
160
161
162

163
164
165

166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199

set testdir [file dirname $argv0]
source $testdir/tester.tcl
ifcapable !fts3 { finish_test ; return }
source $testdir/malloc_common.tcl
source $testdir/fts3_common.tcl

# Ensure the lookaside buffer is disabled for these tests.
#
sqlite3 db test.db
sqlite3_db_config_lookaside db 0 0 0


set sqlite_fts3_enable_parentheses 1
set DO_MALLOC_TEST 1





# Test organization:
#

# fts3_malloc-1.*: Test CREATE and DROP table statements.
#
#

#-------------------------------------------------------------------------
# This proc is used to test a single SELECT statement. Parameter $name is
# passed a name for the test case (i.e. "fts3_malloc-1.4.1") and parameter
# $sql is passed the text of the SELECT statement. Parameter $result is
# set to the expected output if the SELECT statement is successfully
# executed using [db eval].
................................................................................
# the SELECT statement succeeds and the results match those specified
# by parameter $result, or (b) TCL throws an "out of memory" error.
#
# If DO_MALLOC_TEST is defined and set to zero, then the SELECT statement
# is executed just once. In this case the test case passes if the results
# match the expected results passed via parameter $result.
#


proc do_passive_test {name sql catchres} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  if {$::DO_MALLOC_TEST} {
    set answers [list {1 {out of memory}} $catchres]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list $catchres]
    set modes [list 0 nofail]
  }
  set str [join $answers " OR "]

  foreach {nRepeat zName} $modes {
    for {set iFail 1} 1 {incr iFail} {
      if {$::DO_MALLOC_TEST} {sqlite3_memdebug_fail $iFail -repeat $nRepeat}
................................................................................
      }
      do_test $name.$zName.$iFail [list set {} $res] $str
      set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign]
      if {$nFail==0} break
    }
  }
}

proc do_select_test {name sql result} {
  do_passive_test $name $sql [list 0 $result]
}

proc do_error_test {name sql error} {
  do_passive_test $name $sql [list 1 $error]
}

#-------------------------------------------------------------------------
# Test a single write to the database. In this case a  "write" is a 
# DELETE, UPDATE or INSERT statement.
#
# If OOM testing is performed, there are several acceptable outcomes:
#
................................................................................
# transaction is rolled back, non-zero is returned.
#
# Parameter $name is the name to use for the test case (or test cases).
# The second parameter, $tbl, should be the name of the database table
# being modified. Parameter $sql contains the SQL statement to test.
#
proc do_write_test {name tbl sql} {
  if {![info exists ::DO_MALLOC_TEST]} { set ::DO_MALLOC_TEST 1 }

  # Figure out an statement to get a checksum for table $tbl.
  db eval "SELECT * FROM $tbl" V break
  set cksumsql "SELECT md5sum([join [concat rowid $V(*)] ,]) FROM $tbl"

  # Calculate the initial table checksum.
  set cksum1 [db one $cksumsql]



  if {$::DO_MALLOC_TEST } {
    set answers [list {1 {out of memory}} {0 {}}]
    set modes [list 100000 transient 1 persistent]
  } else {
    set answers [list {0 {}}]
    set modes [list 0 nofail]
................................................................................

proc normal_list {l} {
  set ret [list]
  foreach elem $l {lappend ret $elem}
  set ret
}



do_write_test fts3_malloc-1.1 sqlite_master {
  CREATE VIRTUAL TABLE ft1 USING fts3(a, b)

}
do_write_test fts3_malloc-1.2 sqlite_master {
  CREATE VIRTUAL TABLE ft2 USING fts3([a], [b]);
}
do_write_test fts3_malloc-1.3 sqlite_master {
  CREATE VIRTUAL TABLE ft3 USING fts3('a', "b");
}
do_write_test fts3_malloc-1.4 sqlite_master {
  CREATE VIRTUAL TABLE ft4 USING fts3(`a`, 'fred''s column');
}
do_error_test fts3_malloc-1.5 {
  CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown)
} {unknown tokenizer: unknown}
do_write_test fts3_malloc-1.6 sqlite_master {
  CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter)
}

# Test the xConnect/xDisconnect methods:
db eval { ATTACH 'test2.db' AS aux }
do_write_test fts3_malloc-1.6 aux.sqlite_master {
  CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);
}
do_write_test fts3_malloc-1.6 aux.sqlite_master {
  CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c);
}



do_test fts3_malloc-2.0 {
  execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) }
  for {set ii 1} {$ii < 32} {incr ii} {
    set a [list]
    set b [list]
    if {$ii & 0x01} {lappend a one   ; lappend b neung}