Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a segfault that can occur when querying an empty FTS3 table. Also restore the rowid/docid conflict handling to work as it did in version 3.6.20. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
c022f66b5a65aa54d5ebd55cfe941118 |
User & Date: | dan 2009-12-01 12:00:22.000 |
Context
2009-12-01
| ||
13:48 | Open a savepoint within the FTS3 optimize() function. (check-in: 4924fbb244 user: dan tags: trunk) | |
12:00 | Fix a segfault that can occur when querying an empty FTS3 table. Also restore the rowid/docid conflict handling to work as it did in version 3.6.20. (check-in: c022f66b5a user: dan tags: trunk) | |
2009-11-30
| ||
19:48 | Test coverage improvements in the FTS3 porter stemmer. (check-in: 6d112bfd53 user: drh tags: trunk) | |
Changes
Changes to ext/fts3/fts3_write.c.
︙ | ︙ | |||
579 580 581 582 583 584 585 | /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. ** Which is a problem, since "rowid" and "docid" are aliases for the ** same value. For example: ** ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); ** | | | > > > > > > | 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 | /* There is a quirk here. The users INSERT statement may have specified ** a value for the "rowid" field, for the "docid" field, or for both. ** Which is a problem, since "rowid" and "docid" are aliases for the ** same value. For example: ** ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); ** ** In FTS3, this is an error. It is an error to specify non-NULL values ** for both docid and some other rowid alias. */ if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ if( SQLITE_NULL==sqlite3_value_type(apVal[0]) && SQLITE_NULL!=sqlite3_value_type(apVal[1]) ){ /* A rowid/docid conflict. */ return SQLITE_ERROR; } rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); if( rc!=SQLITE_OK ) return rc; } /* Execute the statement to insert the record. Set *piDocid to the ** new docid value. */ |
︙ | ︙ | |||
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 | int nAlloc = 0; /* Allocated size of aBuffer buffer */ int rc = SQLITE_OK; /* Return code */ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ | > > > > > | 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 | int nAlloc = 0; /* Allocated size of aBuffer buffer */ int rc = SQLITE_OK; /* Return code */ int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); /* If there are zero segments, this function is a no-op. This scenario ** comes about only when reading from an empty database. */ if( nSegment==0 ) goto finished; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ |
︙ | ︙ |
Changes to test/e_fts3.test.
︙ | ︙ | |||
12 13 14 15 16 17 18 | # This file implements tests to verify the "testable statements" in the # fts3.in document. # set testdir [file dirname $argv0] source $testdir/tester.tcl | > > | < < < > > > > > > > > > > > > > | | | | | | | | | | | | > > > > > > > > > | | | | | | | | > > > > > > > | > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 | # This file implements tests to verify the "testable statements" in the # fts3.in document. # set testdir [file dirname $argv0] source $testdir/tester.tcl # If this build does not include FTS3, skip the tests in this file. # ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl # Procs used to make the tests in this file easier to read. # proc ddl_test {tn ddl} { uplevel [list do_write_test e_fts3-$tn sqlite_master $ddl] } proc write_test {tn tbl sql} { uplevel [list do_write_test e_fts3-$tn $tbl $sql] } proc read_test {tn sql result} { uplevel [list do_select_test e_fts3-$tn $sql $result] } #------------------------------------------------------------------------- # The body of the following [foreach] block contains test cases to verify # that the example code in fts3.html works as expected. The tests run three # times, with different values for DO_MALLOC_TEST. # # DO_MALLOC_TEST=0: Run tests with no OOM errors. # DO_MALLOC_TEST=1: Run tests with transient OOM errors. # DO_MALLOC_TEST=2: Run tests with persistent OOM errors. # foreach DO_MALLOC_TEST {0 1 2} { # Reset the database and database connection. If this iteration of the # [foreach] loop is testing with OOM errors, disable the lookaside buffer. # db close file delete -force test.db test.db-journal sqlite3 db test.db if {$DO_MALLOC_TEST} { sqlite3_db_config_lookaside db 0 0 0 } ########################################################################## # Test the example CREATE VIRTUAL TABLE statements in section 1.1 # of fts3.in. # ddl_test 1.1.1.1 {CREATE VIRTUAL TABLE data USING fts3()} read_test 1.1.1.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} ddl_test 1.1.2.1 { CREATE VIRTUAL TABLE pages USING fts3(title, keywords, body) } read_test 1.1.2.2 { PRAGMA table_info(pages) } {0 title {} 0 {} 0 1 keywords {} 0 {} 0 2 body {} 0 {} 0} ddl_test 1.1.3.1 { CREATE VIRTUAL TABLE mail USING fts3( subject VARCHAR(256) NOT NULL, body TEXT CHECK(length(body)<10240) ) } read_test 1.1.3.2 { PRAGMA table_info(mail) } {0 subject {} 0 {} 0 1 body {} 0 {} 0} # A very large string. Used to test if the constraint on column "body" of # table "mail" is enforced (it should not be - FTS3 tables do not support # constraints). set largetext [string repeat "the quick brown fox " 5000] write_test 1.1.3.3 mail_content { INSERT INTO mail VALUES(NULL, $largetext) } read_test 1.1.3.4 { SELECT subject IS NULL, length(body) FROM mail } [list 1 100000] ddl_test 1.1.4.1 { CREATE VIRTUAL TABLE papers USING fts3(author, document, tokenize=porter) } read_test 1.1.4.2 { PRAGMA table_info(papers) } {0 author {} 0 {} 0 1 document {} 0 {} 0} ddl_test 1.1.5.1 { CREATE VIRTUAL TABLE simpledata USING fts3(tokenize=simple) } read_test 1.1.5.2 { PRAGMA table_info(simpledata) } {0 content {} 0 {} 0} ifcapable icu { ddl_test 1.1.6.1 { CREATE VIRTUAL TABLE names USING fts3(a, b, tokenize=icu en_AU) } read_test 1.1.6.2 { PRAGMA table_info(names) } {0 a {} 0 {} 0 1 b {} 0 {} 0} } ddl_test 1.1.7.1 {DROP TABLE data} ddl_test 1.1.7.2 {DROP TABLE pages} ddl_test 1.1.7.3 {DROP TABLE mail} ddl_test 1.1.7.4 {DROP TABLE papers} ddl_test 1.1.7.5 {DROP TABLE simpledata} read_test 1.1.7.6 {SELECT * FROM sqlite_master} {} # The following is not one of the examples in section 1.1. It tests # specifying an FTS3 table with no module arguments using a slightly # different syntax. ddl_test 1.1.8.1 {CREATE VIRTUAL TABLE data USING fts3;} read_test 1.1.8.2 {PRAGMA table_info(data)} {0 content {} 0 {} 0} ddl_test 1.1.8.3 {DROP TABLE data} ########################################################################## ########################################################################## # Test the examples in section 1.2 (populating fts3 tables) # ddl_test 1.2.1.1 { CREATE VIRTUAL TABLE pages USING fts3(title, body); } write_test 1.2.1.2 pages_content { INSERT INTO pages(docid, title, body) VALUES(53, 'Home Page', 'SQLite is a software...'); } read_test 1.2.1.3 { SELECT docid, * FROM pages } {53 {Home Page} {SQLite is a software...}} write_test 1.2.1.4 pages_content { INSERT INTO pages(title, body) VALUES('Download', 'All SQLite source code...'); } read_test 1.2.1.5 { SELECT docid, * FROM pages } {53 {Home Page} {SQLite is a software...} 54 Download {All SQLite source code...}} write_test 1.2.1.6 pages_content { UPDATE pages SET title = 'Download SQLite' WHERE rowid = 54 } read_test 1.2.1.7 { SELECT docid, * FROM pages } {53 {Home Page} {SQLite is a software...} 54 {Download SQLite} {All SQLite source code...}} write_test 1.2.1.8 pages_content { DELETE FROM pages } read_test 1.2.1.9 { SELECT docid, * FROM pages } {} do_error_test fts3-1.2.1.10 { INSERT INTO pages(rowid, docid, title, body) VALUES(1, 2, 'A title', 'A document body'); } {SQL logic error or missing database} # Test the optimize() function example: if 0 { ddl_test 1.2.2.1 { CREATE VIRTUAL TABLE docs USING fts3 } write_test 1.2.2.2 docs_content { INSERT INTO docs VALUES('Others translate the first clause as'); } write_test 1.2.2.3 docs_content { INSERT INTO docs VALUES('"which is for Solomon," meaning that'); } write_test 1.2.2.4 docs_content { INSERT INTO docs VALUES('the book is dedicated to Solomon.'); } read_test 1.2.2.5 { SELECT count(*) FROM docs_segdir } {3} set DO_MALLOC_TEST 1 write_test 1.2.2.6 docs_segdir { INSERT INTO docs SELECT * FROM (SELECT optimize(docs) FROM docs LIMIT 1) WHERE 0; } read_test 1.2.2.7 { SELECT count(*) FROM docs_segdir } {1} } ########################################################################## # Test the examples in section 1.3 (querying FTS3 tables) # ddl_test 1.3.1.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } read_test 1.3.1.2 { SELECT * FROM mail WHERE rowid = 15; -- Fast. Rowid lookup. SELECT * FROM mail WHERE body MATCH 'sqlite'; -- Fast. Full-text query. SELECT * FROM mail WHERE mail MATCH 'search'; -- Fast. Full-text query. SELECT * FROM mail WHERE rowid BETWEEN 15 AND 20; -- Slow. Linear scan. SELECT * FROM mail WHERE subject = 'database'; -- Slow. Linear scan. SELECT * FROM mail WHERE subject MATCH 'database'; -- Fast. Full-text query. } {} ddl_test 1.3.1.3 { DROP TABLE mail } ddl_test 1.3.2.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } write_test 1.3.2.2 mail_content { INSERT INTO mail(docid, subject, body) VALUES(1, 'software feedback', 'found it too slow') } write_test 1.3.2.3 mail_content { INSERT INTO mail(docid, subject, body) VALUES(2, 'software feedback', 'no feedback') } write_test 1.3.2.4 mail_content { INSERT INTO mail(docid, subject, body) VALUES(3, 'slow lunch order', 'was a software problem') } read_test 1.3.2.5 { SELECT * FROM mail WHERE subject MATCH 'software' } {{software feedback} {found it too slow} {software feedback} {no feedback}} read_test 1.3.2.6 { SELECT * FROM mail WHERE body MATCH 'feedback' } {{software feedback} {no feedback}} read_test 1.3.2.7 { SELECT * FROM mail WHERE mail MATCH 'software' } {{software feedback} {found it too slow} {software feedback} {no feedback} {slow lunch order} {was a software problem}} read_test 1.3.2.7 { SELECT * FROM mail WHERE mail MATCH 'slow' } {{software feedback} {found it too slow} {slow lunch order} {was a software problem}} ddl_test 1.3.2.8 { DROP TABLE mail } ddl_test 1.3.3.1 { CREATE VIRTUAL TABLE docs USING fts3(content) } read_test 1.3.3.2 { SELECT * FROM docs WHERE docs MATCH 'sqlite' } {} read_test 1.3.3.3 { SELECT * FROM docs WHERE docs.docs MATCH 'sqlite' } {} read_test 1.3.3.4 { SELECT * FROM docs WHERE main.docs.docs MATCH 'sqlite' } {} do_error_test e_fts3-1.3.3.5 { SELECT * FROM docs WHERE main.docs MATCH 'sqlite' } {no such column: main.docs} ddl_test 1.3.2.8 { DROP TABLE docs } ########################################################################## ########################################################################## # Test the examples in section 3 (full-text index queries). # ddl_test 1.4.1.1 { CREATE VIRTUAL TABLE docs USING fts3(title, body) } foreach {tn title body} { 2 "linux driver" "a device" 3 "driver" "linguistic trick" 4 "problems" "linux problems" 5 "linux" "big problems" 6 "linux driver" "a device driver problem" 7 "good times" "applications for linux" 8 "not so good" "linux applications" 9 "alternative" "linoleum appliances" 10 "no L I N" "to be seen" } { write_test 1.4.1.$tn docs_content { INSERT INTO docs VALUES($title,$body) } set R($tn) [list $title $body] } read_test 1.4.1.11 { SELECT * FROM docs WHERE docs MATCH 'linux' } [concat $R(2) $R(4) $R(5) $R(6) $R(7) $R(8)] read_test 1.4.1.12 { SELECT * FROM docs WHERE docs MATCH 'lin*' } [concat $R(2) $R(3) $R(4) $R(5) $R(6) $R(7) $R(8) $R(9)] read_test 1.4.1.13 { SELECT * FROM docs WHERE docs MATCH 'title:linux problems' } [concat $R(5)] read_test 1.4.1.14 { SELECT * FROM docs WHERE body MATCH 'title:linux driver' } [concat $R(6)] read_test 1.4.1.15 { SELECT * FROM docs WHERE docs MATCH '"linux applications"' } [concat $R(8)] read_test 1.4.1.16 { SELECT * FROM docs WHERE docs MATCH '"lin* app*"' } [concat $R(8) $R(9)] ddl_test 1.4.1.17 { DROP TABLE docs } ddl_test 1.4.2.1 { CREATE VIRTUAL TABLE docs USING fts3() } write_test 1.4.2.2 docs_content { INSERT INTO docs VALUES( 'SQLite is an ACID compliant embedded relational database management system') } foreach {tn query hit} { 3 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR database'} 1 4 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/6 sqlite'} 1 5 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/5 sqlite'} 0 6 {SELECT * FROM docs WHERE docs MATCH 'database NEAR/2 "ACID compliant"'} 1 7 {SELECT * FROM docs WHERE docs MATCH '"ACID compliant" NEAR/2 sqlite'} 1 8 {SELECT * FROM docs WHERE docs MATCH 'sqlite NEAR/2 acid NEAR/2 relational'} 1 9 {SELECT * FROM docs WHERE docs MATCH 'acid NEAR/2 sqlite NEAR/2 relational'} 0 } { set res [db eval {SELECT * FROM docs WHERE $hit}] read_test 1.4.2.$tn $query $res } ddl_test 1.4.2.10 { DROP TABLE docs } unset R # TODO: Change numbering after here... ########################################################################## # Test the example in section 5 (custom tokenizers). # ddl_test 2.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) } write_test 2.1.2 simple_content { INSERT INTO simple VALUES('Right now they''re very frustrated') } read_test 2.1.3 {SELECT docid FROM simple WHERE simple MATCH 'Frustrated'} {1} read_test 2.1.4 {SELECT docid FROM simple WHERE simple MATCH 'Frustration'} {} ddl_test 2.2.1 { CREATE VIRTUAL TABLE porter USING fts3(tokenize=porter) } write_test 2.2.2 porter_content { INSERT INTO porter VALUES('Right now they''re very frustrated') } read_test 2.2.3 {SELECT docid FROM porter WHERE porter MATCH 'Frustrated'} {1} read_test 2.2.4 {SELECT docid FROM porter WHERE porter MATCH 'Frustration'} {1} ########################################################################## ########################################################################## # Test the examples in section 4 (auxillary functions). # ddl_test 3.1.1 { CREATE VIRTUAL TABLE mail USING fts3(subject, body) } write_test 3.1.2 mail_content { |
︙ | ︙ | |||
159 160 161 162 163 164 165 166 167 | read_test 3.2.4 { SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"' } {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}} #break } finish_test | > > > | 354 355 356 357 358 359 360 361 362 363 364 365 | read_test 3.2.4 { SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"' } {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}} #break } # End of tests of example code in fts3.html #------------------------------------------------------------------------- finish_test |