Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Fix the fts5 integrity-check so that it works with columnsize=0 tables. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5-incompatible |
| Files: | files | file ages | folders |
| SHA1: |
11b887b15eaee57ea2577c763e70494f |
| User & Date: | dan 2015-09-03 11:00:50 |
Context
|
2015-09-03
| ||
| 11:17 | Remove dead code from fts5_index.c. check-in: 8a0a9b01 user: dan tags: fts5-incompatible | |
| 11:00 | Fix the fts5 integrity-check so that it works with columnsize=0 tables. check-in: 11b887b1 user: dan tags: fts5-incompatible | |
| 10:27 | Fix a memory leak in fts5_expr.c. check-in: 399932a1 user: dan tags: fts5-incompatible | |
Changes
Changes to ext/fts5/fts5_storage.c.
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 ... 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 .... 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 |
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
if( rc==SQLITE_OK ){
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( ctx.szCol!=aColSize[i] ) rc = FTS5_CORRUPT;
aTotalSize[i] += ctx.szCol;
}
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
................................................................................
/* Check that the %_docsize and %_content tables contain the expected
** number of rows. */
if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
i64 nRow;
rc = fts5StorageCount(p, "content", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
if( rc==SQLITE_OK ){
i64 nRow;
rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
/* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by
................................................................................
** each table column. This function reads the %_docsize record for the
** specified rowid and populates aCol[] with the results.
**
** An SQLite error code is returned if an error occurs, or SQLITE_OK
** otherwise.
*/
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
int nCol = p->pConfig->nCol;
sqlite3_stmt *pLookup = 0;
int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( rc==SQLITE_OK ){
int bCorrupt = 1;
sqlite3_bind_int64(pLookup, 1, iRowid);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
int nBlob = sqlite3_column_bytes(pLookup, 0);
if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
|
> | > > | > | | | > > > | |
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 ... 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 .... 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 |
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
if( rc==SQLITE_OK ){
int rc2;
while( SQLITE_ROW==sqlite3_step(pScan) ){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
if( pConfig->bColumnsize ){
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
}
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);
if( rc==SQLITE_OK ) rc = rc2;
}
................................................................................
/* Check that the %_docsize and %_content tables contain the expected
** number of rows. */
if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
i64 nRow;
rc = fts5StorageCount(p, "content", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
if( rc==SQLITE_OK && pConfig->bColumnsize ){
i64 nRow;
rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
/* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by
................................................................................
** each table column. This function reads the %_docsize record for the
** specified rowid and populates aCol[] with the results.
**
** An SQLite error code is returned if an error occurs, or SQLITE_OK
** otherwise.
*/
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
int nCol = p->pConfig->nCol; /* Number of user columns in table */
sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
int rc; /* Return Code */
assert( p->pConfig->bColumnsize );
rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( rc==SQLITE_OK ){
int bCorrupt = 1;
sqlite3_bind_int64(pLookup, 1, iRowid);
if( SQLITE_ROW==sqlite3_step(pLookup) ){
const u8 *aBlob = sqlite3_column_blob(pLookup, 0);
int nBlob = sqlite3_column_bytes(pLookup, 0);
if( 0==fts5StorageDecodeSizeArray(aCol, nCol, aBlob, nBlob) ){
|
Changes to ext/fts5/test/fts5columnsize.test.
130 131 132 133 134 135 136 137 138 |
}
do_execsql_test 3.2.1 {
SELECT rowid, fts5_test_columnsize(t4) FROM t4 WHERE t4 MATCH 'a'
} {
1 {-1 0 -1} 2 {-1 0 -1}
}
finish_test
|
> > | > > > > > > > > > > > |
130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 |
}
do_execsql_test 3.2.1 {
SELECT rowid, fts5_test_columnsize(t4) FROM t4 WHERE t4 MATCH 'a'
} {
1 {-1 0 -1} 2 {-1 0 -1}
}
#-------------------------------------------------------------------------
# Test the integrity-check
#
do_execsql_test 4.1.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0);
INSERT INTO t5 VALUES('1 2 3 4');
INSERT INTO t5 VALUES('2 4 6 8');
}
breakpoint
do_execsql_test 4.1.2 {
INSERT INTO t5(t5) VALUES('integrity-check');
}
finish_test
|
Added ext/fts5/test/fts5fault7.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
# 2015 September 3 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #************************************************************************* # # This file is focused on OOM errors. # source [file join [file dirname [info script]] fts5_common.tcl] source $testdir/malloc_common.tcl set testprefix fts5fault2 # If SQLITE_ENABLE_FTS3 is defined, omit this file. ifcapable !fts5 { finish_test return } #------------------------------------------------------------------------- # Test fault-injection on a query that uses xColumnSize() on columnsize=0 # table. # do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0); INSERT INTO t1 VALUES('a b c d e f g'); INSERT INTO t1 VALUES('a b c d'); INSERT INTO t1 VALUES('a b c d e f g h i j'); } fts5_aux_test_functions db do_faultsim_test 1 -faults oom* -body { execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' } } -test { faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM} } finish_test |
Changes to ext/fts5/test/fts5synonym.test.
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 |
if {[string length $w]==1} {
for {set i 2} {$i<=10} {incr i} {
sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
}
}
}
}
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=0, tokenize=tcl);
INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
} {{3 4} {2 10}}
finish_test
|
> | | > > > > > > > > > > > > > > > |
429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 |
if {[string length $w]==1} {
for {set i 2} {$i<=10} {incr i} {
sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
}
}
}
}
do_execsql_test 7.0.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
} {{3 4} {2 10}}
do_execsql_test 7.0.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_execsql_test 7.1.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
} {{3 4} {2 10}}
do_execsql_test 7.1.2 {
INSERT INTO t2(t2) VALUES('integrity-check');
}
finish_test
|