Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
372c1db2475f367d54270d5801aff050 |
| User & Date: | dan 2015-07-08 17:59:08 |
Context
|
2015-07-09
| ||
| 19:02 | Reduce the number of calls to malloc() made by fts5. check-in: 898618cc user: dan tags: trunk | |
|
2015-07-08
| ||
| 17:59 | Fix two problems that could cause fts3 auxiliary functions to occasionally misbehave if used with match expressions that contain both OR and NEAR. check-in: 372c1db2 user: dan tags: trunk | |
| 16:22 | Enhance the pcache1 page cache so that it tries to allocate a block of SQLITE_DEFAULT_PCACHE_INITSZ pages from malloc() on startup, and uses those preallocated pages when possible rather than going to malloc() for each individual page. About a 5% performance increase for some workloads. check-in: 5348ffc3 user: drh tags: trunk | |
Changes
Changes to ext/fts3/fts3.c.
4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 .... 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 .... 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 |
assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
if( p==0 ){
p = aDoclist;
p += sqlite3Fts3GetVarint(p, piDocid);
}else{
fts3PoslistCopy(0, &p);
if( p>=&aDoclist[nDoclist] ){
*pbEof = 1;
}else{
sqlite3_int64 iVar;
p += sqlite3Fts3GetVarint(p, &iVar);
*piDocid += ((bDescIdx ? -1 : 1) * iVar);
}
................................................................................
iDocid = pExpr->iDocid;
pIter = pPhrase->doclist.pList;
if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
int rc = SQLITE_OK;
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int bOr = 0;
u8 bEof = 0;
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
/* Check if this phrase descends from an OR expression node. If not,
** return NULL. Otherwise, the entry that corresponds to docid
** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
** tree that the node is part of has been marked as EOF, but the node
** itself is not EOF, then it may point to an earlier entry. */
pNear = pExpr;
................................................................................
if( bTreeEof ){
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
}
}
if( rc!=SQLITE_OK ) return rc;
pIter = pPhrase->pOrPoslist;
iDocid = pPhrase->iOrDocid;
if( pCsr->bDesc==bDescDoclist ){
bEof = !pPhrase->doclist.nAll ||
(pIter >= (pPhrase->doclist.aAll + pPhrase->doclist.nAll));
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}else{
bEof = !pPhrase->doclist.nAll || (pIter && pIter<=pPhrase->doclist.aAll);
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
int dummy;
sqlite3Fts3DoclistPrev(
bDescDoclist, pPhrase->doclist.aAll, pPhrase->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}
pPhrase->pOrPoslist = pIter;
pPhrase->iOrDocid = iDocid;
if( bEof || iDocid!=pCsr->iPrevId ) pIter = 0;
}
if( pIter==0 ) return SQLITE_OK;
if( *pIter==0x01 ){
pIter++;
pIter += fts3GetVarint32(pIter, &iThis);
}else{
|
> < > > > > > > > > > > > | | | | | | | < > | | | | | | | | < > | | | | | | > | < > > > > > > |
4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 .... 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 .... 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 |
assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) );
if( p==0 ){
p = aDoclist;
p += sqlite3Fts3GetVarint(p, piDocid);
}else{
fts3PoslistCopy(0, &p);
while( p<&aDoclist[nDoclist] && *p==0 ) p++;
if( p>=&aDoclist[nDoclist] ){
*pbEof = 1;
}else{
sqlite3_int64 iVar;
p += sqlite3Fts3GetVarint(p, &iVar);
*piDocid += ((bDescIdx ? -1 : 1) * iVar);
}
................................................................................
iDocid = pExpr->iDocid;
pIter = pPhrase->doclist.pList;
if( iDocid!=pCsr->iPrevId || pExpr->bEof ){
int rc = SQLITE_OK;
int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */
int bOr = 0;
u8 bTreeEof = 0;
Fts3Expr *p; /* Used to iterate from pExpr to root */
Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */
int bMatch;
/* Check if this phrase descends from an OR expression node. If not,
** return NULL. Otherwise, the entry that corresponds to docid
** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the
** tree that the node is part of has been marked as EOF, but the node
** itself is not EOF, then it may point to an earlier entry. */
pNear = pExpr;
................................................................................
if( bTreeEof ){
while( rc==SQLITE_OK && !pNear->bEof ){
fts3EvalNextRow(pCsr, pNear, &rc);
}
}
if( rc!=SQLITE_OK ) return rc;
bMatch = 1;
for(p=pNear; p; p=p->pLeft){
u8 bEof = 0;
Fts3Expr *pTest = p;
Fts3Phrase *pPh;
assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE );
if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight;
assert( pTest->eType==FTSQUERY_PHRASE );
pPh = pTest->pPhrase;
pIter = pPh->pOrPoslist;
iDocid = pPh->iOrDocid;
if( pCsr->bDesc==bDescDoclist ){
bEof = !pPh->doclist.nAll ||
(pIter >= (pPh->doclist.aAll + pPh->doclist.nAll));
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){
sqlite3Fts3DoclistNext(
bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
&pIter, &iDocid, &bEof
);
}
}else{
bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll);
while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){
int dummy;
sqlite3Fts3DoclistPrev(
bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll,
&pIter, &iDocid, &dummy, &bEof
);
}
}
pPh->pOrPoslist = pIter;
pPh->iOrDocid = iDocid;
if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0;
}
if( bMatch ){
pIter = pPhrase->pOrPoslist;
}else{
pIter = 0;
}
}
if( pIter==0 ) return SQLITE_OK;
if( *pIter==0x01 ){
pIter++;
pIter += fts3GetVarint32(pIter, &iThis);
}else{
|
Changes to test/fts3fault.test.
13 14 15 16 17 18 19 20 21 22 23 24 25 26 ... 153 154 155 156 157 158 159 160 161 162 163 164 165 166 ... 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 |
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts3fault
# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }
# Test error handling in the sqlite3Fts3Init() function. This is the
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body {
sqlite3 db test.db
................................................................................
execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
{1 {vtable constructor failed: t1}} \
{1 {SQL logic error or missing database}}
}
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
binary scan $blob $scan($::tcl_platform(byteOrder)) r
return $r
}
................................................................................
execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
execsql "INSERT INTO t8 VALUES('d d d')"
execsql "INSERT INTO t8 VALUES('e e e')"
execsql "INSERT INTO t8(t8) VALUES('optimize')"
faultsim_save_and_close
} {}
do_faultsim_test 8.1 -prep {
faultsim_restore_and_reopen
db func mit mit
} -body {
execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}
do_faultsim_test 8.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
db func mit mit
} -body {
execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
faultsim_test_result {0 3}
|
> > > > > | > |
13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 ... 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 ... 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 |
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts3fault
# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }
if 0 {
# Test error handling in the sqlite3Fts3Init() function. This is the
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body {
sqlite3 db test.db
................................................................................
execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
{1 {vtable constructor failed: t1}} \
{1 {SQL logic error or missing database}}
}
}
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
binary scan $blob $scan($::tcl_platform(byteOrder)) r
return $r
}
................................................................................
execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
execsql "INSERT INTO t8 VALUES('d d d')"
execsql "INSERT INTO t8 VALUES('e e e')"
execsql "INSERT INTO t8(t8) VALUES('optimize')"
faultsim_save_and_close
} {}
do_faultsim_test 8.1 -faults oom-t* -prep {
faultsim_restore_and_reopen
db func mit mit
} -body {
execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}
do_faultsim_test 8.2 -faults oom-t* -prep {
faultsim_restore_and_reopen
db func mit mit
} -body {
execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
faultsim_test_result {0 3}
|
Added test/fts3offsets.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
# 2010 November 02 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # set testdir [file dirname $argv0] source $testdir/tester.tcl # If SQLITE_ENABLE_FTS3 is not defined, omit this file. ifcapable !fts3 { finish_test ; return } set testprefix fts3offsets set sqlite_fts3_enable_parentheses 1 proc extract {offsets text} { set res "" set off [list] foreach {t i s n} $offsets { lappend off [list $s $n] } set off [lsort -integer -index 0 $off] set iOff 0 foreach e $off { foreach {s n} $e {} append res [string range $text $iOff $s-1] append res "(" append res [string range $text $s [expr $s+$n-1]] append res ")" set iOff [expr $s+$n] } append res [string range $text $iOff end] set res } db func extract extract do_execsql_test 1.1.0 { CREATE VIRTUAL TABLE xx USING fts3(x); INSERT INTO xx VALUES('A x x x B C x x'); INSERT INTO xx VALUES('A B C x B x x C'); INSERT INTO xx VALUES('A x x B C x x x'); } do_execsql_test 1.1.1 { SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'; } { 1 {(A) x x x (B) (C) x x} 2 {(A) (B) (C) x (B) x x C} 3 {(A) x x (B) (C) x x x} } do_execsql_test 1.2 { DELETE FROM xx; INSERT INTO xx VALUES('A x x x B C x x'); INSERT INTO xx VALUES('A x x C x x x C'); INSERT INTO xx VALUES('A x x B C x x x'); } do_execsql_test 1.2.1 { SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'; } { 1 {(A) x x x (B) (C) x x} 2 {(A) x x C x x x C} 3 {(A) x x (B) (C) x x x} } do_execsql_test 1.3 { DELETE FROM xx; INSERT INTO xx(rowid, x) VALUES(1, 'A B C'); INSERT INTO xx(rowid, x) VALUES(2, 'A x'); INSERT INTO xx(rowid, x) VALUES(3, 'A B C'); INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B'); INSERT INTO xx(rowid, x) VALUES(5, 'A x x x x x x x x x C'); INSERT INTO xx(rowid, x) VALUES(6, 'A x x x x x x x x x x x B'); INSERT INTO xx(rowid, x) VALUES(7, 'A B C'); } do_execsql_test 1.3.1 { SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)'; } { 1 {(A) (B) (C)} 2 {(A) x} 3 {(A) (B) (C)} 4 {(A) (B) (C) x x x x x x x B} 5 {(A) x x x x x x x x x C} 6 {(A) x x x x x x x x x x x B} 7 {(A) (B) (C)} } do_execsql_test 1.4 { DELETE FROM xx; INSERT INTO xx(rowid, x) VALUES(7, 'A B C'); INSERT INTO xx(rowid, x) VALUES(6, 'A x'); INSERT INTO xx(rowid, x) VALUES(5, 'A B C'); INSERT INTO xx(rowid, x) VALUES(4, 'A B C x x x x x x x B'); INSERT INTO xx(rowid, x) VALUES(3, 'A x x x x x x x x x C'); INSERT INTO xx(rowid, x) VALUES(2, 'A x x x x x x x x x x x B'); INSERT INTO xx(rowid, x) VALUES(1, 'A B C'); } do_execsql_test 1.4.1 { SELECT oid,extract(offsets(xx), x) FROM xx WHERE xx MATCH 'a OR (b NEAR/1 c)' ORDER BY docid DESC; } { 7 {(A) (B) (C)} 6 {(A) x} 5 {(A) (B) (C)} 4 {(A) (B) (C) x x x x x x x B} 3 {(A) x x x x x x x x x C} 2 {(A) x x x x x x x x x x x B} 1 {(A) (B) (C)} } set sqlite_fts3_enable_parentheses 0 finish_test |