Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Fix a problem that occurs when more than 4 synonyms for a term appear within a single row. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5-incompatible |
Files: | files | file ages | folders |
SHA1: |
cd359550bdc2bf7be4c294b60130c9fc |
User & Date: | dan 2015-09-01 18:44:33.194 |
Context
2015-09-02
| ||
08:22 | Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API. (check-in: cf3e45e76d user: dan tags: fts5-incompatible) | |
2015-09-01
| ||
18:44 | Fix a problem that occurs when more than 4 synonyms for a term appear within a single row. (check-in: cd359550bd user: dan tags: fts5-incompatible) | |
18:08 | Add tests for fts5 synonyms implemented by adding extra terms to queries. And fixes for the same. (check-in: dbcb73802b user: dan tags: fts5-incompatible) | |
Changes
Changes to ext/fts5/fts5_expr.c.
︙ | ︙ | |||
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 | int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); | > | > > > > > > > > > > > | 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 | int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int nIter = 0; int nAlloc = 4; int rc = SQLITE_OK; Fts5ExprTerm *p; assert( pTerm->pSynonym ); for(p=pTerm; p; p=p->pSynonym){ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; i64 dummy; rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); if( rc!=SQLITE_OK ) goto synonym_poslist_out; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; goto synonym_poslist_out; } memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; aIter = aNew; } if( sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter])==0 ){ nIter++; } } } assert( *pbDel==0 ); |
︙ | ︙ | |||
443 444 445 446 447 448 449 450 451 452 453 454 455 456 | }else{ *pa = buf.p; *pn = buf.n; *pbDel = 1; } } return rc; } /* ** All individual term iterators in pPhrase are guaranteed to be valid and ** pointing to the same rowid when this function is called. This function | > > | 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 | }else{ *pa = buf.p; *pn = buf.n; *pbDel = 1; } } synonym_poslist_out: if( aIter!=aStatic ) sqlite3_free(aIter); return rc; } /* ** All individual term iterators in pPhrase are guaranteed to be valid and ** pointing to the same rowid when this function is called. This function |
︙ | ︙ |
Changes to ext/fts5/test/fts5synonym.test.
︙ | ︙ | |||
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 | 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') FROM t1 WHERE t1 MATCH $q } $res } finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 | 2 {[5] 1 3 [4] i} {2 2 [v] two [4]} 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1} 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3} 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii} 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]} 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1} } 3 {one OR two OR iii OR 4 OR v} { 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]} 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]} 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]} 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]} 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]} 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]} 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]} 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]} 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]} } } { do_execsql_test 5.1.$tn { SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']') FROM t1 WHERE t1 MATCH $q } $res } #------------------------------------------------------------------------- # Test terms with more than 4 synonyms. # reset_db sqlite3_fts5_create_tokenizer db tcl tcl_create proc tcl_tokenize {tflags text} { foreach {w iStart iEnd} [do_tokenize_split $text] { sqlite3_fts5_token $w $iStart $iEnd if {$tflags=="query" && [string length $w]==1} { for {set i 2} {$i<=10} {incr i} { sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd } } } } do_test 6.0 { execsql { CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl) } foreach {rowid a b} { 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa} 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj} 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii} 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n} 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq} 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu} 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss} 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg} } { execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) } } } {} foreach {tn q res} { 1 {a} { 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]} 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj} 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii} 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq} } 2 {a AND q} { 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]} } 3 {o OR (q AND a)} { 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]} 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq} 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n} 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]} 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg} } } { do_execsql_test 6.1.$tn { SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']') FROM t2 WHERE t2 MATCH $q } $res } do_execsql_test 6.2.1 { INSERT INTO t2(rowid, a, b) VALUES(13, 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy' ); SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')') FROM t2 WHERE t2 MATCH 'x OR y' } { 1 {<yyyy> vvvvv qq oo <yyyyyy> vvvv eee} {ffff uu r qq aaaa} 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq} 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii} 13 {<x> <xx> <xxx> <xxxx> <xxxxx> <xxxxxx> <xxxxxxx>} {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)} } finish_test |