Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts3-prefix-search |
| Files: | files | file ages | folders |
| SHA1: |
b9fb69e55bb05a8819688ee63987f2a0 |
| User & Date: | dan 2011-06-13 17:00:12 |
Context
|
2011-06-13
| ||
| 18:21 | Update trace2.test to account for new FTS queries. (check-in: 2c201292 user: dan tags: fts3-prefix-search) | |
| 17:00 | Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. (check-in: b9fb69e5 user: dan tags: fts3-prefix-search) | |
| 13:48 | Changes to fts3auto.test to test OR, AND and NOT operations. (check-in: e4ab6cdb user: dan tags: fts3-prefix-search) | |
Changes
Changes to ext/fts3/fts3.c.
3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 .... 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 .... 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 |
pTC->pRoot = pRoot;
pTC->pToken = &pPhrase->aToken[i];
pTC->iCol = pPhrase->iColumn;
*pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
}
}else if( pExpr->eType!=FTSQUERY_NOT ){
if( pExpr->eType==FTSQUERY_OR ){
pRoot = pExpr;
**ppOr = pExpr;
(*ppOr)++;
}
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
}
}
}
static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
if( pCsr->nRowAvg==0 ){
................................................................................
/* If bOptOk is true, check if there are any tokens that should be deferred.
*/
if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
Fts3TokenAndCost *aTC;
Fts3Expr **apOr;
aTC = (Fts3TokenAndCost *)sqlite3_malloc(
sizeof(Fts3TokenAndCost) * nToken
+ sizeof(Fts3Expr *) * nOr
);
apOr = (Fts3Expr **)&aTC[nToken];
if( !aTC ){
rc = SQLITE_NOMEM;
}else{
int ii;
................................................................................
Fts3Expr *pExpr, /* Phrase expression */
u32 *aiOut /* Array to write results into (see above) */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
int iCol;
if( pExpr->bDeferred ){
assert( pCsr->nDoc>0 );
for(iCol=0; iCol<pTab->nColumn; iCol++){
aiOut[iCol*3 + 1] = pCsr->nDoc;
aiOut[iCol*3 + 2] = pCsr->nDoc;
}
}else{
rc = fts3EvalGatherStats(pCsr, pExpr);
|
| | > > > > > | | |
3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 .... 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 .... 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 |
pTC->pRoot = pRoot;
pTC->pToken = &pPhrase->aToken[i];
pTC->iCol = pPhrase->iColumn;
*pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl);
}
}else if( pExpr->eType!=FTSQUERY_NOT ){
if( pExpr->eType==FTSQUERY_OR ){
pRoot = pExpr->pLeft;
**ppOr = pRoot;
(*ppOr)++;
}
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc);
if( pExpr->eType==FTSQUERY_OR ){
pRoot = pExpr->pRight;
**ppOr = pRoot;
(*ppOr)++;
}
fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc);
}
}
}
static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){
if( pCsr->nRowAvg==0 ){
................................................................................
/* If bOptOk is true, check if there are any tokens that should be deferred.
*/
if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){
Fts3TokenAndCost *aTC;
Fts3Expr **apOr;
aTC = (Fts3TokenAndCost *)sqlite3_malloc(
sizeof(Fts3TokenAndCost) * nToken
+ sizeof(Fts3Expr *) * nOr * 2
);
apOr = (Fts3Expr **)&aTC[nToken];
if( !aTC ){
rc = SQLITE_NOMEM;
}else{
int ii;
................................................................................
Fts3Expr *pExpr, /* Phrase expression */
u32 *aiOut /* Array to write results into (see above) */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
int iCol;
if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){
assert( pCsr->nDoc>0 );
for(iCol=0; iCol<pTab->nColumn; iCol++){
aiOut[iCol*3 + 1] = pCsr->nDoc;
aiOut[iCol*3 + 2] = pCsr->nDoc;
}
}else{
rc = fts3EvalGatherStats(pCsr, pExpr);
|
Changes to test/fts3auto.test.
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 ... 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 ... 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 ... 323 324 325 326 327 328 329 330 331 332 333 334 335 336 |
set nDoc($iPhrase,$iCol) 0
}
}
foreach key [array names counts] {
set iCol [lindex [split $key ,] 1]
set iPhrase 0
foreach c $counts($key) {
if {$c>0} { incr nHit($iPhrase,$iCol) 1 }
incr nDoc($iPhrase,$iCol) $c
incr iPhrase
}
}
# Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
# contain the output of matchinfo('x') for the document.
#
foreach docid [array names hits] {
set mi [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend mi [lindex $counts($docid,$iCol) $iPhrase]
lappend mi $nDoc($iPhrase,$iCol)
lappend mi $nHit($iPhrase,$iCol)
}
}
set aMatchinfo($docid) $mi
}
# Set up the nullentry output.
#
set nullentry [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol)
}
}
}
proc matching_brackets {expr} {
if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} {
................................................................................
set nullentry [concat $nullleft $nullright]
}
}
}
}
proc do_near_test {tn tbl expr} {
get_near_results $tbl $expr aMatchinfo
set match $expr
set matchinfo_asc [list]
foreach docid [lsort -integer -incr [array names aMatchinfo]] {
lappend matchinfo_asc $docid $aMatchinfo($docid)
................................................................................
set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1}
#--------------------------------------------------------------------------
# Test cases fts3auto-2.* run some simple tests using the
# [do_near_test] proc.
#
foreach {tn create} {
1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)"
2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)"
3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)"
4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)"
5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)"
6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)"
} {
do_test 2.$tn.1 {
catchsql { DROP TABLE t1 }
execsql $create
for {set i 0} {$i<32} {incr i} {
set doc [list]
if {$i&0x01} {lappend doc one}
if {$i&0x02} {lappend doc two}
if {$i&0x04} {lappend doc three}
if {$i&0x08} {lappend doc four}
if {$i&0x10} {lappend doc five}
execsql { INSERT INTO t1 VALUES($doc, null) }
}
} {}
foreach {tn2 expr} {
1 {one}
2 {one NEAR/1 five}
3 {t*}
4 {t* NEAR/0 five}
5 {o* NEAR/1 f*}
6 {one NEAR five NEAR two NEAR four NEAR three}
................................................................................
11 {one AND two OR three}
12 {three OR one AND two}
13 {(three OR one) AND two}
14 {(three OR one) AND two NOT (five NOT four)}
15 {"one two"}
16 {"one two" NOT "three four"}
} {
do_near_test 2.$tn.2.$tn2 t1 $expr
}
}
set sqlite_fts3_enable_parentheses $sfep
finish_test
|
| | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < > | | | | | | | | | > < > > | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 ... 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 ... 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 ... 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 |
set nDoc($iPhrase,$iCol) 0
}
}
foreach key [array names counts] {
set iCol [lindex [split $key ,] 1]
set iPhrase 0
foreach c $counts($key) {
if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
incr nHit($iPhrase,$iCol) $c
incr iPhrase
}
}
if {[info exists ::fts3_deferred] && [llength $expr]==1} {
set phrase [lindex $expr 0]
set rewritten [list]
set partial 0
foreach tok $phrase {
if {[lsearch $::fts3_deferred $tok]>=0} {
lappend rewritten *
} else {
lappend rewritten $tok
set partial 1
}
}
if {$partial==0} {
set tblsize [db one "SELECT count(*) FROM $tbl"]
for {set iCol 0} {$iCol < $nCol} {incr iCol} {
set nHit(0,$iCol) $tblsize
set nDoc(0,$iCol) $tblsize
}
} elseif {$rewritten != $phrase} {
while {[lindex $rewritten end] == "*"} {
set rewritten [lrange $rewritten 0 end-1]
}
while {[lindex $rewritten 0] == "*"} {
set rewritten [lrange $rewritten 1 end]
}
get_single_near_results $tbl [list $rewritten] aRewrite nullentry
foreach docid [array names hits] {
set aMatchinfo($docid) $aRewrite($docid)
}
return
}
}
# Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
# contain the output of matchinfo('x') for the document.
#
foreach docid [array names hits] {
set mi [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend mi [lindex $counts($docid,$iCol) $iPhrase]
lappend mi $nHit($iPhrase,$iCol)
lappend mi $nDoc($iPhrase,$iCol)
}
}
set aMatchinfo($docid) $mi
}
# Set up the nullentry output.
#
set nullentry [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
}
}
}
proc matching_brackets {expr} {
if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} {
................................................................................
set nullentry [concat $nullleft $nullright]
}
}
}
}
proc do_fts3query_test {tn tbl expr} {
get_near_results $tbl $expr aMatchinfo
set match $expr
set matchinfo_asc [list]
foreach docid [lsort -integer -incr [array names aMatchinfo]] {
lappend matchinfo_asc $docid $aMatchinfo($docid)
................................................................................
set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1}
#--------------------------------------------------------------------------
# Test cases fts3auto-2.* run some simple tests using the
# [do_fts3query_test] proc.
#
foreach {tn create} {
1 "fts4(a, b)"
2 "fts4(a, b, order=DESC)"
3 "fts4(a, b, order=ASC)"
4 "fts4(a, b, prefix=1)"
5 "fts4(a, b, order=DESC, prefix=1)"
6 "fts4(a, b, order=ASC, prefix=1)"
} {
break
do_test 2.$tn.1 {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING $create"
for {set i 0} {$i<32} {incr i} {
set doc [list]
if {$i&0x01} {lappend doc one}
if {$i&0x02} {lappend doc two}
if {$i&0x04} {lappend doc three}
if {$i&0x08} {lappend doc four}
if {$i&0x10} {lappend doc five}
execsql { INSERT INTO t1 VALUES($doc, null) }
}
} {}
foreach {tn2 expr} {
1 {one}
2 {one NEAR/1 five}
3 {t*}
4 {t* NEAR/0 five}
5 {o* NEAR/1 f*}
6 {one NEAR five NEAR two NEAR four NEAR three}
................................................................................
11 {one AND two OR three}
12 {three OR one AND two}
13 {(three OR one) AND two}
14 {(three OR one) AND two NOT (five NOT four)}
15 {"one two"}
16 {"one two" NOT "three four"}
} {
do_fts3query_test 2.$tn.2.$tn2 t1 $expr
}
}
#--------------------------------------------------------------------------
# Some test cases involving deferred tokens.
#
proc make_token_deferrable {tbl token} {
set nRow [db one "SELECT count(*) FROM $tbl"]
set pgsz [db one "PRAGMA page_size"]
execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')"
execsql BEGIN
for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
set doc [string repeat "$token " 100]
execsql "INSERT INTO $tbl VALUES(\$doc)"
}
execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')"
execsql COMMIT
}
foreach {tn create} {
1 "fts4(x)"
2 "fts4(x, order=DESC)"
} {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING $create"
do_execsql_test 3.$tn.1 {
INSERT INTO t1 VALUES('a b c d e f g h i j k');
INSERT INTO t1 VALUES('b c d e f g h i j k a');
INSERT INTO t1 VALUES('c d e f g h i j k a b');
INSERT INTO t1 VALUES('d e f g h i j k a b c');
INSERT INTO t1 VALUES('e f g h i j k a b c d');
INSERT INTO t1 VALUES('f g h i j k a b c d e');
INSERT INTO t1 VALUES('a c e g i k');
INSERT INTO t1 VALUES('a d g j');
INSERT INTO t1 VALUES('c a b');
}
make_token_deferrable t1 c
set ::fts3_deferred [list]
foreach {tn2 expr} {
1 {a OR c}
} {
do_fts3query_test 3.$tn.2.$tn2 t1 $expr
}
set ::fts3_deferred [list c]
execsql {
UPDATE t1_segments
SET block = zeroblob(length(block))
WHERE length(block)>10000 AND 0
}
foreach {tn2 expr} {
1 {a NEAR c}
2 {a AND c}
3 {"a c"}
4 {"c a"}
5 {"a c" NEAR/1 g}
6 {"a c" NEAR/0 g}
} {
do_fts3query_test 3.$tn.2.$tn2 t1 $expr
}
}
set sqlite_fts3_enable_parentheses $sfep
finish_test
|
Changes to test/fts3defer2.test.
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}
do_execsql_test 1.2.1 {
SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}
breakpoint
do_execsql_test 1.2.2 {
SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list \
{a b c d [e] [f] [a] x y} \
{0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9]
]
do_execsql_test 1.2.3 {
SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list \
{[a] b c d [e] [f] [a] x y} \
{0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9]
]
do_execsql_test 1.3.1 { DROP TABLE t1 }
#-----------------------------------------------------------------------------
# Test cases fts3defer2-2.* focus specifically on the matchinfo function.
#
|
< | | |
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}
do_execsql_test 1.2.1 {
SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}
do_execsql_test 1.2.2 {
SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list \
{a b c d [e] [f] [a] x y} \
{0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9]
]
do_execsql_test 1.2.3 {
SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list \
{[a] b c d [e] [f] [a] x y} \
{0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9]
]
do_execsql_test 1.3.1 { DROP TABLE t1 }
#-----------------------------------------------------------------------------
# Test cases fts3defer2-2.* focus specifically on the matchinfo function.
#
|