Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix handling of an OOM error in the fts3 offsets() function. Fix a couple of snippet related test cases in e_fts3.test. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA1: |
14dc46a74aafe44c0bf7dffd26268395 |
| User & Date: | dan 2010-01-09 07:33:54 |
Context
|
2010-01-11
| ||
| 12:00 | Modify snippets code to run more efficiently. And to avoid a bug relating to snippets based on full-text queries that contain duplicate terms. check-in: a2b1183d user: dan tags: trunk | |
|
2010-01-09
| ||
| 07:33 | Fix handling of an OOM error in the fts3 offsets() function. Fix a couple of snippet related test cases in e_fts3.test. check-in: 14dc46a7 user: dan tags: trunk | |
|
2010-01-08
| ||
| 23:01 | Update comments in fts3.c to more accurately describe the doclist format. check-in: e424a030 user: drh tags: trunk | |
Changes
Changes to ext/fts3/fts3_snippet.c.
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 |
/* Initialize the contents of sCtx.aTerm[] for column iCol. */
sCtx.iCol = iCol;
sCtx.iTerm = 0;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
if( rc!=SQLITE_OK ) goto offsets_out;
/* Initialize a tokenizer iterator to iterate through column iCol. */
zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
if( rc!=SQLITE_OK ) goto offsets_out;
pC->pTokenizer = pTab->pTokenizer;
rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
while( rc==SQLITE_OK ){
int i; /* Used to loop through terms */
|
| > > > > > > > > > > > > > > |
904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 |
/* Initialize the contents of sCtx.aTerm[] for column iCol. */
sCtx.iCol = iCol;
sCtx.iTerm = 0;
rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
if( rc!=SQLITE_OK ) goto offsets_out;
/* Retreive the text stored in column iCol. If an SQL NULL is stored
** in column iCol, jump immediately to the next iteration of the loop.
** If an OOM occurs while retrieving the data (this can happen if SQLite
** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM
** to the caller.
*/
zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1);
nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
if( zDoc==0 ){
if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){
continue;
}
rc = SQLITE_NOMEM;
goto offsets_out;
}
/* Initialize a tokenizer iterator to iterate through column iCol. */
rc = pMod->xOpen(pTab->pTokenizer, zDoc, nDoc, &pC);
if( rc!=SQLITE_OK ) goto offsets_out;
pC->pTokenizer = pTab->pTokenizer;
rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent);
while( rc==SQLITE_OK ){
int i; /* Used to loop through terms */
|
Changes to test/e_fts3.test.
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 ... 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 ... 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 |
} {{1 0 5 7 1 0 30 7}}
read_test 1.7.1.6 {
SELECT offsets(mail) FROM mail WHERE mail MATCH '"serious mail"'
} {{1 0 28 7 1 1 36 4}}
ddl_test 1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() }
write_test 3.2.2 text_content {
INSERT INTO text VALUES('
During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr.
');
}
read_test 1.7.2.3 {
SELECT snippet(text) FROM text WHERE text MATCH 'cold'
} {{<b>...</b> elsewhere, minimum temperature 17-20oC. <b>Cold</b> to very <b>cold</b> on mountaintops, minimum <b>...</b>}}
read_test 1.7.2.4 {
SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"'
} {{... 2-3oC drops. Cool in the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] ...}}
##########################################################################
# Test the example in section 5 (custom tokenizers).
#
ddl_test 1.8.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) }
write_test 1.8.1.2 simple_content {
INSERT INTO simple VALUES('Right now they''re very frustrated')
................................................................................
error_test 2.1.6 {
SELECT snippet(a) FROM t1 WHERE a MATCH 'one'
} {illegal first argument to snippet}
error_test 2.1.7 {
SELECT snippet() FROM t1 WHERE a MATCH 'one'
} {unable to use function snippet in the requested context}
error_test 2.1.8 {
SELECT snippet(a, b, 'A', 'B', 'C') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
................................................................................
set DO_MALLOC_TEST 0
ddl_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea. So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 {
SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <b>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.4 {
SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <i>miles</b> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.5 {
SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b> Down to a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towers were <b>...</b>}}
read_test 5.6 {
SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXX Down to a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towers were XXX}}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0
|
| | | | | | | | |
407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 ... 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 ... 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 |
} {{1 0 5 7 1 0 30 7}}
read_test 1.7.1.6 {
SELECT offsets(mail) FROM mail WHERE mail MATCH '"serious mail"'
} {{1 0 28 7 1 1 36 4}}
ddl_test 1.7.2.1 { CREATE VIRTUAL TABLE text USING fts3() }
write_test 1.7.2.2 text_content {
INSERT INTO text VALUES('
During 30 Nov-1 Dec, 2-3oC drops. Cool in the upper portion, minimum temperature 14-16oC and cool elsewhere, minimum temperature 17-20oC. Cold to very cold on mountaintops, minimum temperature 6-12oC. Northeasterly winds 15-30 km/hr. After that, temperature increases. Northeasterly winds 15-30 km/hr.
');
}
read_test 1.7.2.3 {
SELECT snippet(text) FROM text WHERE text MATCH 'cold'
} {{<b>...</b>cool elsewhere, minimum temperature 17-20oC. <b>Cold</b> to very <b>cold</b> on mountaintops, minimum temperature 6<b>...</b>}}
read_test 1.7.2.4 {
SELECT snippet(text, '[', ']', '...') FROM text WHERE text MATCH '"min* tem*"'
} {{...the upper portion, [minimum] [temperature] 14-16oC and cool elsewhere, [minimum] [temperature] 17-20oC. Cold...}}
##########################################################################
# Test the example in section 5 (custom tokenizers).
#
ddl_test 1.8.1.1 { CREATE VIRTUAL TABLE simple USING fts3(tokenize=simple) }
write_test 1.8.1.2 simple_content {
INSERT INTO simple VALUES('Right now they''re very frustrated')
................................................................................
error_test 2.1.6 {
SELECT snippet(a) FROM t1 WHERE a MATCH 'one'
} {illegal first argument to snippet}
error_test 2.1.7 {
SELECT snippet() FROM t1 WHERE a MATCH 'one'
} {unable to use function snippet in the requested context}
error_test 2.1.8 {
SELECT snippet(a, b, 'A', 'B', 'C', 'D', 'E') FROM t1 WHERE a MATCH 'one'
} {wrong number of arguments to function snippet()}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test the effect of an OOM error while installing the FTS3 module (i.e.
# opening a database handle). This case was not tested by the OOM testing
# of the document examples above.
................................................................................
set DO_MALLOC_TEST 0
ddl_test 5.1 { CREATE VIRTUAL TABLE t5 USING fts3(x) }
write_test 5.2 t5_content {
INSERT INTO t5 VALUES('In Xanadu did Kubla Khan A stately pleasure-dome decree Where Alph, the sacred river, ran Through caverns measureless to man Down to a sunless sea. So twice five miles of fertile ground With walls and towers were girdled round : And there were gardens bright with sinuous rills, Where blossomed many an incense-bearing tree ; And here were forests ancient as the hills, Enfolding sunny spots of greenery.');
}
read_test 5.3 {
SELECT snippet(t5) FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea. So twice five <b>miles</b> of fertile ground With walls and towers<b>...</b>}}
read_test 5.4 {
SELECT snippet(t5, '<i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea. So twice five <i>miles</b> of fertile ground With walls and towers<b>...</b>}}
read_test 5.5 {
SELECT snippet(t5, '<i>', '</i>') FROM t5 WHERE t5 MATCH 'miles'
} {{<b>...</b>to a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towers<b>...</b>}}
read_test 5.6 {
SELECT snippet(t5, '<i>', '</i>', 'XXX') FROM t5 WHERE t5 MATCH 'miles'
} {{XXXto a sunless sea. So twice five <i>miles</i> of fertile ground With walls and towersXXX}}
#-------------------------------------------------------------------------
#-------------------------------------------------------------------------
# Test that an empty MATCH expression returns an empty result set. As
# does passing a NULL value as a MATCH expression.
#
set DO_MALLOC_TEST 0
|