Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add further tests for fts5 backend. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
09dabb3b9e140eec6cfda83bcb86b6b9 |
User & Date: | dan 2015-01-21 16:10:59.815 |
Context
2015-01-21
| ||
17:20 | Merge trunk changes with this branch. (check-in: f8699a1a3b user: dan tags: fts5) | |
16:10 | Add further tests for fts5 backend. (check-in: 09dabb3b9e user: dan tags: fts5) | |
06:36 | Merge latest trunk changes with this branch. (check-in: b3348b1e07 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
260 261 262 263 264 265 266 267 268 269 270 271 272 273 | static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } # define FTS5_CORRUPT fts5Corrupt() #else # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB #endif typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5ChunkIter Fts5ChunkIter; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5MultiSegIter Fts5MultiSegIter; typedef struct Fts5NodeIter Fts5NodeIter; | > > > > > > > | 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 | static int fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } # define FTS5_CORRUPT fts5Corrupt() #else # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB #endif /* ** Each time a blob is read from the %_data table, it is padded with this ** many zero bytes. This makes it easier to decode the various record formats ** without overreading if the records are corrupt. */ #define FTS5_DATA_ZERO_PADDING 8 typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5ChunkIter Fts5ChunkIter; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5MultiSegIter Fts5MultiSegIter; typedef struct Fts5NodeIter Fts5NodeIter; |
︙ | ︙ | |||
712 713 714 715 716 717 718 | int nByte = sqlite3_blob_bytes(p->pReader); if( pBuf ){ fts5BufferZero(pBuf); fts5BufferGrow(&rc, pBuf, nByte); aOut = pBuf->p; pBuf->n = nByte; }else{ | > | | 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 | int nByte = sqlite3_blob_bytes(p->pReader); if( pBuf ){ fts5BufferZero(pBuf); fts5BufferGrow(&rc, pBuf, nByte); aOut = pBuf->p; pBuf->n = nByte; }else{ int nSpace = nByte + FTS5_DATA_ZERO_PADDING; pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nSpace+sizeof(Fts5Data)); if( pRet ){ pRet->n = nByte; aOut = pRet->p = (u8*)&pRet[1]; pRet->nRef = 1; } } |
︙ | ︙ | |||
4535 4536 4537 4538 4539 4540 4541 | static void fts5DecodeFunction( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args (always 2) */ sqlite3_value **apVal /* Function arguments */ ){ i64 iRowid; /* Rowid for record being decoded */ int iIdx,iSegid,iHeight,iPgno; /* Rowid components */ | | > > | > > > > > | 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 | static void fts5DecodeFunction( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args (always 2) */ sqlite3_value **apVal /* Function arguments */ ){ i64 iRowid; /* Rowid for record being decoded */ int iIdx,iSegid,iHeight,iPgno; /* Rowid components */ const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ int rc = SQLITE_OK; /* Return code */ int nSpace = 0; assert( nArg==2 ); memset(&s, 0, sizeof(Fts5Buffer)); iRowid = sqlite3_value_int64(apVal[0]); n = sqlite3_value_bytes(apVal[1]); aBlob = sqlite3_value_blob(apVal[1]); nSpace = n + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; memcpy(a, aBlob, n); fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){ int i = 0; i64 iPrev; if( n>0 ){ |
︙ | ︙ | |||
4583 4584 4585 4586 4587 4588 4589 | if( iHeight==0 ){ int iTermOff = 0; int iRowidOff = 0; int iOff; int nKeep = 0; | > | | > > > > | 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 | if( iHeight==0 ){ int iTermOff = 0; int iRowidOff = 0; int iOff; int nKeep = 0; if( n>=4 ){ iRowidOff = fts5GetU16(&a[0]); iTermOff = fts5GetU16(&a[2]); }else{ sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt"); goto decode_out; } if( iRowidOff ){ iOff = iRowidOff; }else if( iTermOff ){ iOff = iTermOff; }else{ iOff = n; |
︙ | ︙ | |||
4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 | ); } } fts5NodeIterFree(&ss); } } if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); }else{ sqlite3_result_error_code(pCtx, rc); } fts5BufferFree(&s); } | > > | 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 | ); } } fts5NodeIterFree(&ss); } } decode_out: sqlite3_free(a); if( rc==SQLITE_OK ){ sqlite3_result_text(pCtx, (const char*)s.p, s.n, SQLITE_TRANSIENT); }else{ sqlite3_result_error_code(pCtx, rc); } fts5BufferFree(&s); } |
︙ | ︙ | |||
4689 4690 4691 4692 4693 4694 4695 | idx = sqlite3_value_int(apVal[1]); iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0); sqlite3_result_int64(pCtx, iRowid); } }else { sqlite3_result_error(pCtx, "first arg to fts5_rowid() must be 'segment' " | | | 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 | idx = sqlite3_value_int(apVal[1]); iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0); sqlite3_result_int64(pCtx, iRowid); } }else { sqlite3_result_error(pCtx, "first arg to fts5_rowid() must be 'segment' " "or 'start-of-index'" , -1 ); } } } /* |
︙ | ︙ |
Changes to ext/fts5/test/fts5fault1.test.
︙ | ︙ | |||
27 28 29 30 31 32 33 | # 1: CREATE VIRTUAL TABLE # 2: INSERT statement # 3: DELETE statement # 4: MATCH expressions # # | | | 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | # 1: CREATE VIRTUAL TABLE # 2: INSERT statement # 3: DELETE statement # 4: MATCH expressions # # if 0 { faultsim_save_and_close do_faultsim_test 1 -prep { faultsim_restore_and_reopen } -body { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') } } -test { |
︙ | ︙ | |||
103 104 105 106 107 108 109 110 111 112 113 114 115 116 | } -body " execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' } " -test " faultsim_test_result {[list 0 $res]} " } #------------------------------------------------------------------------- # The following tests use a larger database populated with random data. # # The database page size is set to 512 bytes and the FTS5 page size left # at the default 1000 bytes. This means that reading a node may require # pulling an overflow page from disk, which is an extra opportunity for # an error to occur. | > > > | 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | } -body " execsql { SELECT rowid FROM t2 WHERE t2 MATCH '$expr' } " -test " faultsim_test_result {[list 0 $res]} " } } #------------------------------------------------------------------------- # The following tests use a larger database populated with random data. # # The database page size is set to 512 bytes and the FTS5 page size left # at the default 1000 bytes. This means that reading a node may require # pulling an overflow page from disk, which is an extra opportunity for # an error to occur. |
︙ | ︙ | |||
172 173 174 175 176 177 178 | sqlite3 db test.db } -body { execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') } } -test { faultsim_test_result [list 0 {}] } | > > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 | sqlite3 db test.db } -body { execsql { INSERT INTO x1 VALUES('a b c d', 'e f g h') } } -test { faultsim_test_result [list 0 {}] } do_faultsim_test 5.5.1 -faults oom* -body { execsql { SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=1 } } -test { faultsim_test_result [list 0 1] } do_faultsim_test 5.5.2 -faults oom* -body { execsql { SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid=10 } } -test { faultsim_test_result [list 0 1] } do_faultsim_test 5.5.3 -faults oom* -body { execsql { SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = ( SELECT min(rowid) FROM x1_data WHERE rowid>20 ) } } -test { faultsim_test_result [list 0 1] } do_faultsim_test 5.5.4 -faults oom* -body { execsql { SELECT count(fts5_decode(rowid, block)) FROM x1_data WHERE rowid = ( SELECT max(rowid) FROM x1_data ) } } -test { faultsim_test_result [list 0 1] } finish_test #------------------------------------------------------------------------- # reset_db do_execsql_test 6.0 { CREATE VIRTUAL TABLE x1 USING fts5(x); INSERT INTO x1(x1, rank) VALUES('automerge', 0); |
︙ | ︙ | |||
202 203 204 205 206 207 208 | INSERT INTO x1 VALUES('a b c'); -- 15 SELECT count(*) FROM x1_data; } {17} faultsim_save_and_close | | > > > > > > > > > > | 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 | INSERT INTO x1 VALUES('a b c'); -- 15 SELECT count(*) FROM x1_data; } {17} faultsim_save_and_close do_faultsim_test 6.1 -faults oom* -prep { faultsim_restore_and_reopen } -body { execsql { INSERT INTO x1 VALUES('d e f') } } -test { faultsim_test_result [list 0 {}] if {$testrc==0} { set nCnt [db one {SELECT count(*) FROM x1_data}] if {$nCnt!=3} { error "expected 3 entries but there are $nCnt" } } } #------------------------------------------------------------------------- do_faultsim_test 7.0 -faults oom* -prep { catch { db close } } -body { sqlite3 db test.db } -test { faultsim_test_result [list 0 {}] [list 1 {}] } finish_test |
Added ext/fts5/test/fts5rowid.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | # 2014 Dec 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # Tests of the scalar fts5_rowid() and fts5_decode() functions. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5rowid do_catchsql_test 1.1 { SELECT fts5_rowid() } {1 {should be: fts5_rowid(subject, ....)}} do_catchsql_test 1.2 { SELECT fts5_rowid('segment') } {1 {should be: fts5_rowid('segment', idx, segid, height, pgno))}} do_execsql_test 1.3 { SELECT fts5_rowid('segment', 1, 1, 1, 1) } {4503670494330881} do_catchsql_test 1.4 { SELECT fts5_rowid('start-of-index'); } {1 {should be: fts5_rowid('start-of-index', idx)}} do_execsql_test 1.5 { SELECT fts5_rowid('start-of-index', 1); } {4503668346847232} do_catchsql_test 1.4 { SELECT fts5_rowid('nosucharg'); } {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} #------------------------------------------------------------------------- # Tests of the fts5_decode() function. # reset_db do_execsql_test 2.1 { CREATE VIRTUAL TABLE x1 USING fts5(a, b); INSERT INTO x1(x1, rank) VALUES('pgsz', 32); } {} proc rnddoc {n} { set map [list 0 a 1 b 2 c 3 d 4 e 5 f 6 g 7 h 8 i 9 j] set doc [list] for {set i 0} {$i < $n} {incr i} { lappend doc [string map $map [format %.3d [expr int(rand()*100)]]] } set doc } db func rnddoc rnddoc do_execsql_test 2.2 { WITH r(a, b) AS ( SELECT rnddoc(6), rnddoc(6) UNION ALL SELECT rnddoc(6), rnddoc(6) FROM r ) INSERT INTO x1 SELECT * FROM r LIMIT 10000; } set res [db one {SELECT count(*) FROM x1_data}] do_execsql_test 2.3 { SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res do_execsql_test 2.4 { UPDATE x1_data SET block = X''; SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res do_execsql_test 2.5 { INSERT INTO x1(x1, rank) VALUES('pgsz', 1024); INSERT INTO x1(x1) VALUES('rebuild'); } set res [db one {SELECT count(*) FROM x1_data}] do_execsql_test 2.6 { SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res do_execsql_test 2.7 { UPDATE x1_data SET block = X''; SELECT count(fts5_decode(rowid, block)) FROM x1_data; } $res #------------------------------------------------------------------------- # Tests with very large tokens. # set strlist [list \ "[string repeat x 400]" \ "[string repeat x 300][string repeat w 100]" \ "[string repeat x 300][string repeat y 100]" \ "[string repeat x 300][string repeat z 600]" \ ] do_test 3.0 { execsql { BEGIN; CREATE VIRTUAL TABLE x2 USING fts5(a); } foreach str $strlist { execsql { INSERT INTO x2 VALUES($str) } } execsql COMMIT } {} for {set tn 0} {$tn<[llength $strlist]} {incr tn} { set str [lindex $strlist $tn] do_execsql_test 3.1.$tn { SELECT rowid FROM x2 WHERE x2 MATCH $str } [expr $tn+1] } set res [db one {SELECT count(*) FROM x2_data}] do_execsql_test 3.2 { SELECT count(fts5_decode(rowid, block)) FROM x2_data; } $res finish_test |