Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add some tests for OR, AND and NOT operations to fts3rnd.test. Add tests to check that errors are returned when bad arguments are passed to fts3 functions snippet, offsets and optimize. Minor fix for the same |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
5811df3f0412598d189d46b58de4deff |
User & Date: | dan 2009-12-07 12:34:52.000 |
Context
2009-12-07
| ||
14:48 | Version 3.6.21 release candidate 2. (check-in: 78f6baffb0 user: drh tags: trunk) | |
12:34 | Add some tests for OR, AND and NOT operations to fts3rnd.test. Add tests to check that errors are returned when bad arguments are passed to fts3 functions snippet, offsets and optimize. Minor fix for the same (check-in: 5811df3f04 user: dan tags: trunk) | |
2009-12-06
| ||
03:35 | Enhanced detection of database corruption in btree.c:allocateSpace(). (check-in: 5a511f9887 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
642 643 644 645 646 647 648 | int nString = 0; int nCol = 0; char *zCsr; int nDb; int nName; #ifdef SQLITE_TEST | | | 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 | int nString = 0; int nCol = 0; char *zCsr; int nDb; int nName; #ifdef SQLITE_TEST const char *zTestParam = 0; if( strncmp(argv[argc-1], "test:", 5)==0 ){ zTestParam = argv[argc-1]; argc--; } #endif const char *zTokenizer = 0; /* Name of tokenizer to use */ |
︙ | ︙ | |||
1466 1467 1468 1469 1470 1471 1472 | fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; } | | < | 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 | fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } break; } default: assert( mergetype==MERGE_POS_NEAR || mergetype==MERGE_NEAR ); { char *aTmp = 0; char **ppPos = 0; if( mergetype==MERGE_POS_NEAR ){ ppPos = &p; aTmp = sqlite3_malloc(2*(n1+n2)); if( !aTmp ){ return SQLITE_NOMEM; |
︙ | ︙ | |||
1502 1503 1504 1505 1506 1507 1508 | fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } sqlite3_free(aTmp); break; } | < < < | 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 | fts3PoslistCopy(0, &p2); fts3GetDeltaVarint2(&p2, pEnd2, &i2); } } sqlite3_free(aTmp); break; } } *pnBuffer = (int)(p-aBuffer); return SQLITE_OK; } /* |
︙ | ︙ | |||
2081 2082 2083 2084 2085 2086 2087 | sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB | | | | | > > > > > > > > | > | | | | | | 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 | sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) ){ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); sqlite3_result_error(pContext, zErr, -1); sqlite3_free(zErr); return SQLITE_ERROR; } memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); *ppCsr = pRet; return SQLITE_OK; } /* ** Implementation of the snippet() function for FTS3 */ static void fts3SnippetFunc( sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of apVal[] array */ sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ const char *zStart = "<b>"; const char *zEnd = "</b>"; const char *zEllipsis = "<b>...</b>"; /* There must be at least one argument passed to this function (otherwise ** the non-overloaded version would have been called instead of this one). */ assert( nVal>=1 ); if( nVal>4 ){ sqlite3_result_error(pContext, "wrong number of arguments to function snippet()", -1); return; } if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; switch( nVal ){ case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); } sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis); } /* ** Implementation of the offsets() function for FTS3 |
︙ | ︙ |
Changes to test/e_fts3.test.
︙ | ︙ | |||
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 | } proc write_test {tn tbl sql} { uplevel [list do_write_test e_fts3-$tn $tbl $sql] } proc read_test {tn sql result} { uplevel [list do_select_test e_fts3-$tn $sql $result] } #------------------------------------------------------------------------- # The body of the following [foreach] block contains test cases to verify # that the example code in fts3.html works as expected. The tests run three # times, with different values for DO_MALLOC_TEST. # # DO_MALLOC_TEST=0: Run tests with no OOM errors. # DO_MALLOC_TEST=1: Run tests with transient OOM errors. # DO_MALLOC_TEST=2: Run tests with persistent OOM errors. # | > > > | | 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | } proc write_test {tn tbl sql} { uplevel [list do_write_test e_fts3-$tn $tbl $sql] } proc read_test {tn sql result} { uplevel [list do_select_test e_fts3-$tn $sql $result] } proc error_test {tn sql result} { uplevel [list do_error_test e_fts3-$tn $sql $result] } #------------------------------------------------------------------------- # The body of the following [foreach] block contains test cases to verify # that the example code in fts3.html works as expected. The tests run three # times, with different values for DO_MALLOC_TEST. # # DO_MALLOC_TEST=0: Run tests with no OOM errors. # DO_MALLOC_TEST=1: Run tests with transient OOM errors. # DO_MALLOC_TEST=2: Run tests with persistent OOM errors. # foreach DO_MALLOC_TEST [lrange {0 1 2} 0 end] { # Reset the database and database connection. If this iteration of the # [foreach] loop is testing with OOM errors, disable the lookaside buffer. # db close file delete -force test.db test.db-journal sqlite3 db test.db |
︙ | ︙ | |||
431 432 433 434 435 436 437 | SELECT docid FROM porter WHERE porter MATCH 'Frustration' } {1} } # End of tests of example code in fts3.html #------------------------------------------------------------------------- | > > > | > > > > > > > > > > > > > > > > > > > > > > > > | 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 | SELECT docid FROM porter WHERE porter MATCH 'Frustration' } {1} } # End of tests of example code in fts3.html #------------------------------------------------------------------------- #------------------------------------------------------------------------- # Test that errors in the arguments passed to the snippet and offsets # functions are handled correctly. # set DO_MALLOC_TEST 0 ddl_test 2.1.1 { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } write_test 2.1.2 t1_content { INSERT INTO t1 VALUES('one two three', x'A1B2C3D4E5F6'); } error_test 2.1.3 { SELECT offsets(a) FROM t1 WHERE a MATCH 'one' } {illegal first argument to offsets} error_test 2.1.4 { SELECT offsets(b) FROM t1 WHERE a MATCH 'one' } {illegal first argument to offsets} error_test 2.1.5 { SELECT optimize(a) FROM t1 LIMIT 1 } {illegal first argument to optimize} error_test 2.1.6 { SELECT snippet(a) FROM t1 WHERE a MATCH 'one' } {illegal first argument to snippet} error_test 2.1.7 { SELECT snippet() FROM t1 WHERE a MATCH 'one' } {unable to use function snippet in the requested context} error_test 2.1.8 { SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one' } {wrong number of arguments to function snippet()} finish_test |
Changes to test/fts3rnd.test.
︙ | ︙ | |||
65 66 67 68 69 70 71 | set cols {a b c} set iCol [expr int(rand()*3)] set doc [generate_doc [expr int((rand()*100))]] lset ::t1($rowid) $iCol $doc execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid" } | < < | < < < < < | < < < < < < | 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | set cols {a b c} set iCol [expr int(rand()*3)] set doc [generate_doc [expr int((rand()*100))]] lset ::t1($rowid) $iCol $doc execsql "UPDATE t1 SET [lindex $cols $iCol] = \$doc WHERE rowid = \$rowid" } proc simple_phrase {zPrefix} { set ret [list] set pattern "*[string map {* \[a-z\]} $zPrefix]*" foreach {key value} [array get ::t1] { if {[string match $pattern $value]} { lappend ret $key } } lsort -integer $ret } proc simple_near {termlist nNear} { set ret [list] foreach {key value} [array get ::t1] { foreach v $value { set l [lsearch -exact -all $v [lindex $termlist 0]] |
︙ | ︙ | |||
117 118 119 120 121 122 123 124 125 126 127 128 129 130 | } } } lsort -unique -integer $ret } foreach nodesize {50 500 1000 2000} { catch { array unset ::t1 } # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 | } } } lsort -unique -integer $ret } # The following three procs: # # setup_not A B # setup_or A B # setup_and A B # # each take two arguments. Both arguments must be lists of integer values # sorted by value. The return value is the list produced by evaluating # the equivalent of "A op B", where op is the FTS3 operator NOT, OR or # AND. # proc setop_not {A B} { foreach b $B { set n($b) {} } set ret [list] foreach a $A { if {![info exists n($a)]} {lappend ret $a} } return $ret } proc setop_or {A B} { lsort -integer -uniq [concat $A $B] } proc setop_and {A B} { foreach b $B { set n($b) {} } set ret [list] foreach a $A { if {[info exists n($a)]} {lappend ret $a} } return $ret } set sqlite_fts3_enable_parentheses 1 foreach nodesize {50 500 1000 2000} { catch { array unset ::t1 } # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } |
︙ | ︙ | |||
158 159 160 161 162 163 164 | # is the same as the result obtained by scanning the contents of the Tcl # array for each term. # for {set i 0} {$i < 10} {incr i} { set term [random_term] do_test fts3rnd-1.$nodesize.$iTest.1.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $term } | | | | | | > > > > > > > > > > > > > | | > > | > > > > > > > > > > > > > | | > > > > > > > > > > > > > > > > > > > > > | > | | | | > | 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 | # is the same as the result obtained by scanning the contents of the Tcl # array for each term. # for {set i 0} {$i < 10} {incr i} { set term [random_term] do_test fts3rnd-1.$nodesize.$iTest.1.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $term } } [simple_phrase $term] } # This time, use the first two characters of each term as a term prefix # to query for. Test that querying the Tcl array produces the same results # as querying the FTS3 table for the prefix. # for {set i 0} {$i < 10} {incr i} { set prefix [string range [random_term] 0 1] set match "${prefix}*" do_test fts3rnd-1.$nodesize.$iTest.2.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_phrase $match] } # Similar to the above, except for phrase queries. # for {set i 0} {$i < 10} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" do_test fts3rnd-1.$nodesize.$iTest.3.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_phrase $term] } # Three word phrases. # for {set i 0} {$i < 10} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" do_test fts3rnd-1.$nodesize.$iTest.4.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_phrase $term] } # Three word phrases made up of term-prefixes. # for {set i 0} {$i < 10} {incr i} { set query "[string range [random_term] 0 1]* " append query "[string range [random_term] 0 1]* " append query "[string range [random_term] 0 1]*" set match "\"$query\"" do_test fts3rnd-1.$nodesize.$iTest.5.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_phrase $query] } # A NEAR query with terms as the arguments. # for {set i 0} {$i < 10} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] do_test fts3rnd-1.$nodesize.$iTest.6.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_near $terms 10] } # A 3-way NEAR query with terms as the arguments. # for {set i 0} {$i < 10} {incr i} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] set fts3 [execsql { SELECT docid FROM t1 WHERE t1 MATCH $match }] do_test fts3rnd-1.$nodesize.$iTest.7.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [simple_near $terms $nNear] } # Set operations on simple term queries. # foreach {tn op proc} { 8 OR setop_or 9 NOT setop_not 10 AND setop_and } { for {set i 0} {$i < 10} {incr i} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [$proc [simple_phrase $term1] [simple_phrase $term2]] } } # Set operations on NEAR queries. # foreach {tn op proc} { 8 OR setop_or 9 NOT setop_not 10 AND setop_and } { for {set i 0} {$i < 10} {incr i} { set term1 [random_term] set term2 [random_term] set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" do_test fts3rnd-1.$nodesize.$iTest.$tn.$i { execsql { SELECT docid FROM t1 WHERE t1 MATCH $match } } [$proc \ [simple_near [list $term1 $term2] 10] \ [simple_near [list $term3 $term4] 10] ] } } } } finish_test |