/ Check-in [eee921a9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Extra tests for coverage of fts3 code.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1:eee921a99e69a9cd868a89de620bf47c4e26e4b5
User & Date: dan 2009-12-12 13:16:10
Context
2009-12-12
13:58
Rename tkt-d82e3f3721.txt to use the (correct) .test suffix. check-in: 68cccd62 user: drh tags: trunk
13:16
Extra tests for coverage of fts3 code. check-in: eee921a9 user: dan tags: trunk
09:51
Add coverage test cases for fts3. check-in: 8fcb0478 user: dan tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_write.c.

1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
....
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
....
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
















1854
1855
1856
1857
1858
1859
1860
....
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
  char *z,                        /* Pointer to buffer containing block data */
  int n                           /* Size of buffer z in bytes */
){
  sqlite3_stmt *pStmt;
  int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pStmt, 1, iBlock);
    rc = sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
    if( rc==SQLITE_OK ){
      sqlite3_step(pStmt);
      rc = sqlite3_reset(pStmt);
    }
  }
  return rc;
}

/* 
** Insert a record into the %_segdir table.
*/
................................................................................
  int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pStmt, 1, iLevel);
    sqlite3_bind_int(pStmt, 2, iIdx);
    sqlite3_bind_int64(pStmt, 3, iStartBlock);
    sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
    sqlite3_bind_int64(pStmt, 5, iEndBlock);
    rc = sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
    if( rc==SQLITE_OK ){
      sqlite3_step(pStmt);
      rc = sqlite3_reset(pStmt);
    }
  }
  return rc;
}

/*
** Return the size of the common prefix (if any) shared by zPrev and
** zNext, in bytes. For example, 
................................................................................
}

/*
** sqlite3Fts3SegReaderIterate() callback used when merging multiple 
** segments to create a single, larger segment.
*/
static int fts3MergeCallback(
  Fts3Table *p,
  void *pContext,
  char *zTerm,
  int nTerm,
  char *aDoclist,
  int nDoclist
){
  SegmentWriter **ppW = (SegmentWriter **)pContext;
  return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
}

















/*
** This function is used to iterate through a contiguous set of terms 
** stored in the full-text index. It merges data contained in one or 
** more segments to support this.
**
** The second argument is passed an array of pointers to SegReader objects
................................................................................
  */
  if( rc==SQLITE_OK ){
    void *c = (void *)&pWriter;   /* SegReaderIterate() callback context */
    Fts3SegFilter f;              /* SegReaderIterate() parameters */

    memset(&f, 0, sizeof(Fts3SegFilter));
    f.flags = FTS3_SEGMENT_REQUIRE_POS;
    rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3MergeCallback, c);
  }
  assert( pWriter || rc!=SQLITE_OK );

  /* If no errors have occured, flush the SegmentWriter object to the
  ** database. Then delete the SegmentWriter and Fts3SegReader objects
  ** allocated by this function.
  */







|
<
|
|
<







 







|
<
|
|
<







 







|
|
|
|
|
|




>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|







1224
1225
1226
1227
1228
1229
1230
1231

1232
1233

1234
1235
1236
1237
1238
1239
1240
....
1252
1253
1254
1255
1256
1257
1258
1259

1260
1261

1262
1263
1264
1265
1266
1267
1268
....
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
....
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
  char *z,                        /* Pointer to buffer containing block data */
  int n                           /* Size of buffer z in bytes */
){
  sqlite3_stmt *pStmt;
  int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pStmt, 1, iBlock);
    sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);

    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);

  }
  return rc;
}

/* 
** Insert a record into the %_segdir table.
*/
................................................................................
  int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int(pStmt, 1, iLevel);
    sqlite3_bind_int(pStmt, 2, iIdx);
    sqlite3_bind_int64(pStmt, 3, iStartBlock);
    sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
    sqlite3_bind_int64(pStmt, 5, iEndBlock);
    sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);

    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);

  }
  return rc;
}

/*
** Return the size of the common prefix (if any) shared by zPrev and
** zNext, in bytes. For example, 
................................................................................
}

/*
** sqlite3Fts3SegReaderIterate() callback used when merging multiple 
** segments to create a single, larger segment.
*/
static int fts3MergeCallback(
  Fts3Table *p,                   /* FTS3 Virtual table handle */
  void *pContext,                 /* Pointer to SegmentWriter* to write with */
  char *zTerm,                    /* Term to write to the db */
  int nTerm,                      /* Number of bytes in zTerm */
  char *aDoclist,                 /* Doclist associated with zTerm */
  int nDoclist                    /* Number of bytes in doclist */
){
  SegmentWriter **ppW = (SegmentWriter **)pContext;
  return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
}

/*
** sqlite3Fts3SegReaderIterate() callback used when flushing the contents
** of the pending-terms hash table to the database.
*/
static int fts3FlushCallback(
  Fts3Table *p,                   /* FTS3 Virtual table handle */
  void *pContext,                 /* Pointer to SegmentWriter* to write with */
  char *zTerm,                    /* Term to write to the db */
  int nTerm,                      /* Number of bytes in zTerm */
  char *aDoclist,                 /* Doclist associated with zTerm */
  int nDoclist                    /* Number of bytes in doclist */
){
  SegmentWriter **ppW = (SegmentWriter **)pContext;
  return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist);
}

/*
** This function is used to iterate through a contiguous set of terms 
** stored in the full-text index. It merges data contained in one or 
** more segments to support this.
**
** The second argument is passed an array of pointers to SegReader objects
................................................................................
  */
  if( rc==SQLITE_OK ){
    void *c = (void *)&pWriter;   /* SegReaderIterate() callback context */
    Fts3SegFilter f;              /* SegReaderIterate() parameters */

    memset(&f, 0, sizeof(Fts3SegFilter));
    f.flags = FTS3_SEGMENT_REQUIRE_POS;
    rc = sqlite3Fts3SegReaderIterate(p, &pReader, 1, &f, fts3FlushCallback, c);
  }
  assert( pWriter || rc!=SQLITE_OK );

  /* If no errors have occured, flush the SegmentWriter object to the
  ** database. Then delete the SegmentWriter and Fts3SegReader objects
  ** allocated by this function.
  */

Changes to test/fts3cov.test.

159
160
161
162
163
164
165
166


































167
168
    SELECT 4, '''O well, bright dame, may you command'    UNION ALL
    SELECT 5, 'The service of Sir Leoline;'               UNION ALL
    SELECT 2, 'And gladly our stout chivalry'             UNION ALL
    SELECT 7, 'Will he send forth, and friends withal,'   UNION ALL
    SELECT 8, 'To guide and guard you safe and free'      UNION ALL
    SELECT 6, 'Home to your noble father''s hall.'''
}



































finish_test









>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
    SELECT 4, '''O well, bright dame, may you command'    UNION ALL
    SELECT 5, 'The service of Sir Leoline;'               UNION ALL
    SELECT 2, 'And gladly our stout chivalry'             UNION ALL
    SELECT 7, 'Will he send forth, and friends withal,'   UNION ALL
    SELECT 8, 'To guide and guard you safe and free'      UNION ALL
    SELECT 6, 'Home to your noble father''s hall.'''
}

#-------------------------------------------------------------------------
# When building the internal tree structure for each segment b-tree, FTS3
# assumes that the content of each internal node will be less than
# $nodesize bytes, where $nodesize is the advisory node size. If this turns
# out to be untrue, then an extra buffer must be malloc'd for each term.
# This test case tests these paths and the effects of said mallocs failing
# by inserting insert a document with some fairly large terms into a
# full-text table with a very small node-size. 
#
do_test fts3cov-5.1 {
  execsql {
    CREATE VIRTUAL TABLE t4 USING fts3(x);
    INSERT INTO t4(t4) VALUES('nodesize=24');
  }
} {}
set DO_MALLOC_TEST 1
do_write_test fts3cov-5.2 t4_content {
  INSERT INTO t4
    SELECT 'ItisanancientMarinerAndhestoppethoneofthreeAA' UNION ALL
    SELECT 'ItisanancientMarinerAndhestoppethoneofthreeBB' UNION ALL
    SELECT 'ItisanancientMarinerAndhestoppethoneofthreeCC' UNION ALL
    SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstAA' UNION ALL
    SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstBB' UNION ALL
    SELECT 'BythylonggreybeardandglitteringeyeNowwhereforestoppstCC'
}
do_test fts3cov-5.3 {
  execsql { INSERT INTO t4 VALUES('extra!') }
} {}
do_write_test fts3cov-5.2 t4_segments {
  INSERT INTO t4(t4) VALUES('optimize')
}



finish_test

Changes to test/fts3rnd.test.

90
91
92
93
94
95
96

97
98
99

100
101
102
103
104
105
106
...
137
138
139
140
141
142
143
144




145
146


147


148
149

150
151
152
153
154
155
156
...
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
...
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307

expr srand(0)

# Generate a vocabulary of nVocab words. Each word is 3 characters long.
#
set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z}
for {set i 0} {$i < $nVocab} {incr i} {

  set    word [lindex $lChar [expr int(rand()*26)]]
  append word [lindex $lChar [expr int(rand()*26)]]
  append word [lindex $lChar [expr int(rand()*26)]]

  lappend lVocab $word
}

proc random_term {} {
  lindex $::lVocab [expr {int(rand()*$::nVocab)}]
}

................................................................................
  set doc  [generate_doc [expr int((rand()*100))]]
  lset ::t1($rowid) $iCol $doc
  execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid"
}

proc simple_phrase {zPrefix} {
  set ret [list]
  set pattern "*[string map {* \[a-z\]} $zPrefix]*"




  foreach {key value} [array get ::t1] {
    if {[string match $pattern $value]} { lappend ret $key }


  }


  lsort -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]

  foreach {key value} [array get ::t1] {
    foreach v $value {

      set l [lsearch -exact -all $v [lindex $termlist 0]]
................................................................................
    }

    # This time, use the first two characters of each term as a term prefix
    # to query for. Test that querying the Tcl array produces the same results
    # as querying the FTS3 table for the prefix.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set prefix [string range [random_term] 0 1]
      set match "${prefix}*"
      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $match]
    }

    # Similar to the above, except for phrase queries.
................................................................................
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $term]
    }

    # Three word phrases made up of term-prefixes.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set    query "[string range [random_term] 0 1]* "
      append query "[string range [random_term] 0 1]* "
      append query "[string range [random_term] 0 1]*"

      set match "\"$query\""
      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $query]
    }








>


|
>







 







<
>
>
>
>

<
>
>
|
>
>
|

>







 







|







 







|
|
|







90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
...
139
140
141
142
143
144
145

146
147
148
149
150

151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
...
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
...
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316

expr srand(0)

# Generate a vocabulary of nVocab words. Each word is 3 characters long.
#
set lChar {a b c d e f g h i j k l m n o p q r s t u v w x y z}
for {set i 0} {$i < $nVocab} {incr i} {
  set len [expr int(rand()*3)+2]
  set    word [lindex $lChar [expr int(rand()*26)]]
  append word [lindex $lChar [expr int(rand()*26)]]
  if {$len>2} { append word [lindex $lChar [expr int(rand()*26)]] }
  if {$len>3} { append word [lindex $lChar [expr int(rand()*26)]] }
  lappend lVocab $word
}

proc random_term {} {
  lindex $::lVocab [expr {int(rand()*$::nVocab)}]
}

................................................................................
  set doc  [generate_doc [expr int((rand()*100))]]
  lset ::t1($rowid) $iCol $doc
  execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid"
}

proc simple_phrase {zPrefix} {
  set ret [list]


  set reg [string map {* {[^ ]*}} $zPrefix]
  set reg " $reg "

  foreach {key value} [array get ::t1] {

    foreach col $value {
      if {[regexp $reg " $col "]} {lappend ret $key}
    }
  }

  lsort -uniq -integer $ret
}

proc simple_near {termlist nNear} {
  set ret [list]

  foreach {key value} [array get ::t1] {
    foreach v $value {

      set l [lsearch -exact -all $v [lindex $termlist 0]]
................................................................................
    }

    # This time, use the first two characters of each term as a term prefix
    # to query for. Test that querying the Tcl array produces the same results
    # as querying the FTS3 table for the prefix.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set prefix [string range [random_term] 0 end-1]
      set match "${prefix}*"
      do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $match]
    }

    # Similar to the above, except for phrase queries.
................................................................................
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $term]
    }

    # Three word phrases made up of term-prefixes.
    #
    for {set i 0} {$i < $nRep} {incr i} {
      set    query "[string range [random_term] 0 end-1]* "
      append query "[string range [random_term] 0 end-1]* "
      append query "[string range [random_term] 0 end-1]*"

      set match "\"$query\""
      do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
        SELECT docid FROM t1 WHERE t1 MATCH $match
      } [simple_phrase $query]
    }