Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix problems with prefix queries in fts5. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
3b5758c647530bd5c2e68d0ee3e9f58a |
User & Date: | dan 2015-10-27 17:48:57.642 |
Context
2015-10-27
| ||
20:04 | Have contentless and external content fts5 tables ignore "OR REPLACE" conflict handling. (check-in: a85c2a4758 user: dan tags: trunk) | |
17:48 | Fix problems with prefix queries in fts5. (check-in: 3b5758c647 user: dan tags: trunk) | |
13:35 | Provide hints to the storage engine using the sqlite3BtreeCursorHint() interface when compiling with SQLITE_ENABLE_CURSOR_HINTS. (check-in: 45d3539e15 user: drh tags: trunk) | |
Changes
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 | int iOff = pIter->iLeafOffset; /* Offset to read at */ int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; pIter->iLeafOffset = iOff; } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; | > | 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 | int iOff = pIter->iLeafOffset; /* Offset to read at */ int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; pIter->iLeafOffset = iOff; assert_nc( pIter->nPos>=0 ); } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ int iOff = pIter->iLeafOffset; |
︙ | ︙ | |||
1668 1669 1670 1671 1672 1673 1674 | pIter->iLeafPgno--; pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( pIter->pSeg->iSegid, pIter->iLeafPgno )); if( pNew ){ /* iTermLeafOffset may be equal to szLeaf if the term is the last ** thing on the page - i.e. the first rowid is on the following page. | | | > | < | | > | 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 | pIter->iLeafPgno--; pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( pIter->pSeg->iSegid, pIter->iLeafPgno )); if( pNew ){ /* iTermLeafOffset may be equal to szLeaf if the term is the last ** thing on the page - i.e. the first rowid is on the following page. ** In this case leave pIter->pLeaf==0, this iterator is at EOF. */ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ assert( pIter->pLeaf==0 ); if( pIter->iTermLeafOffset<pNew->szLeaf ){ pIter->pLeaf = pNew; pIter->iLeafOffset = pIter->iTermLeafOffset; } }else{ int iRowidOff; iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ pIter->pLeaf = pNew; pIter->iLeafOffset = iRowidOff; } |
︙ | ︙ | |||
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 | ** this block is particularly performance critical, so equivalent ** code is inlined. */ int nSz; assert( p->rc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; } } } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } | > | 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 | ** this block is particularly performance critical, so equivalent ** code is inlined. */ int nSz; assert( p->rc==SQLITE_OK ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; assert_nc( pIter->nPos>=0 ); } } } } } #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } |
︙ | ︙ | |||
2055 2056 2057 2058 2059 2060 2061 | return; }else if( bEndOfPage ){ do { fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ | > | > > | 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 | return; }else if( bEndOfPage ){ do { fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ iPgidx = pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ nKeep = 0; iTermOff = iOff; n = pIter->pLeaf->nn; iOff += fts5GetVarint32(&a[iOff], nNew); break; } } }while( 1 ); } |
︙ | ︙ | |||
4371 4372 4373 4374 4375 4376 4377 | if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; int i; i64 iLastRowid = 0; Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; | | | 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 | if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; int i; i64 iLastRowid = 0; Fts5IndexIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; Fts5Buffer doclist; int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) ){ i64 iRowid = fts5MultiIterRowid(p1); |
︙ | ︙ | |||
5578 5579 5580 5581 5582 5583 5584 | if( n>0 ){ iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); } while( iOff<n ){ int nPos; | | | > | 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 | if( n>0 ){ iOff = sqlite3Fts5GetVarint(a, (u64*)&iDocid); sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); } while( iOff<n ){ int nPos; int bDel; iOff += fts5GetPoslistSize(&a[iOff], &nPos, &bDel); sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " nPos=%d%s", nPos, bDel?"*":""); iOff += fts5DecodePoslist(pRc, pBuf, &a[iOff], MIN(n-iOff, nPos)); if( iOff<n ){ i64 iDelta; iOff += sqlite3Fts5GetVarint(&a[iOff], (u64*)&iDelta); iDocid += iDelta; sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " id=%lld", iDocid); } |
︙ | ︙ |
Changes to ext/fts5/test/fts5integrity.test.
1 2 3 4 5 6 7 8 9 10 11 | # 2015 Jan 13 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # | | | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 | # 2015 Jan 13 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests focused on the integrity-check procedure. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5integrity # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { |
︙ | ︙ | |||
98 99 100 101 102 103 104 105 106 107 | INSERT INTO aa_content VALUES(23, ''); INSERT INTO aa(aa) VALUES('integrity-check'); } {1 {database disk image is malformed}} #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r} #exit finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | INSERT INTO aa_content VALUES(23, ''); INSERT INTO aa(aa) VALUES('integrity-check'); } {1 {database disk image is malformed}} #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM zz_data} {puts $r} #exit execsql { ROLLBACK } #------------------------------------------------------------------------- # Test that integrity-check works on a reasonably large db with many # different terms. # Document generator command. proc rnddoc {n} { set doc [list] for {set i 0} {$i<$n} {incr i} { lappend doc [format %.5d [expr int(rand()*10000)]] } return $doc } db func rnddoc rnddoc expr srand(0) do_execsql_test 5.0 { CREATE VIRTUAL TABLE gg USING fts5(a, prefix="1,2,3"); INSERT INTO gg(gg, rank) VALUES('pgsz', 256); INSERT INTO gg VALUES(rnddoc(20)); INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; INSERT INTO gg SELECT rnddoc(20) FROM gg; } do_execsql_test 5.1 { INSERT INTO gg(gg) VALUES('integrity-check'); } do_execsql_test 5.2 { INSERT INTO gg(gg) VALUES('optimize'); } breakpoint do_execsql_test 5.3 { INSERT INTO gg(gg) VALUES('integrity-check'); } finish_test |
Added ext/fts5/test/fts5query.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | # 2015 October 27 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # This file implements regression tests for SQLite library. The # focus of this script is testing the FTS5 module. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5query # If SQLITE_ENABLE_FTS5 is defined, omit this file. ifcapable !fts5 { finish_test return } for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { reset_db do_test 1.$tn.1 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); BEGIN; } foreach x [list aaa bbb ccc ddd eee fff ggg hhh iii jjj] { set doc [string repeat "$x " 30] execsql { INSERT INTO t1 VALUES($doc) } } execsql COMMIT } {} do_execsql_test 1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } set ret 1 foreach x [list a b c d e f g h i j] { do_execsql_test 1.$tn.3.$ret { SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; } $ret incr ret } } for {set tn 1 ; set pgsz 64} {$tn<32} {incr tn; incr pgsz 16} { reset_db do_test 2.$tn.1 { execsql { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz); BEGIN; } foreach x [list bbb ddd fff hhh jjj lll nnn ppp rrr ttt] { set doc [string repeat "$x " 30] execsql { INSERT INTO t1 VALUES($doc) } } execsql COMMIT } {} do_execsql_test 1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } set ret 1 foreach x [list a c e g i k m o q s u] { do_execsql_test 2.$tn.3.$ret { SELECT rowid FROM t1 WHERE t1 MATCH $x || '*'; } {} incr ret } } finish_test |
Changes to ext/fts5/tool/showfts5.tcl.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 | #------------------------------------------------------------------------- # Process command line arguments. # proc usage {} { puts stderr "usage: $::argv0 ?OPTIONS? database table" puts stderr "" puts stderr " -nterm (count number of terms in each segment)" puts stderr "" exit 1 } set O(nterm) 0 if {[llength $argv]<2} usage foreach a [lrange $argv 0 end-2] { switch -- $a { -nterm { set O(nterm) 1 } default { usage } } } | > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | #------------------------------------------------------------------------- # Process command line arguments. # proc usage {} { puts stderr "usage: $::argv0 ?OPTIONS? database table" puts stderr "" puts stderr " -nterm (count number of terms in each segment)" puts stderr " -segments (output segment contents)" puts stderr "" exit 1 } set O(nterm) 0 set O(segments) 0 if {[llength $argv]<2} usage foreach a [lrange $argv 0 end-2] { switch -- $a { -nterm { set O(nterm) 1 } -segments { set O(segments) 1 } default { usage } } } |
︙ | ︙ | |||
73 74 75 76 77 78 79 80 81 82 83 84 | puts [format " % -28s nTerm=%d" $seg $nTerm] } else { puts [format " % -28s" $seg] } } } } | > > > > > > > | 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | puts [format " % -28s nTerm=%d" $seg $nTerm] } else { puts [format " % -28s" $seg] } } } } if {$O(segments)} { puts "" db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id>10" { puts $d } } |