Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add extra tests for corrupt database handling in fts5. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
41449f7a0b5da6332eef48386c91ef63 |
User & Date: | dan 2015-04-24 15:56:09.379 |
Context
2015-04-24
| ||
19:41 | Add the "unindexed" column option to fts5. (check-in: 8630996134 user: dan tags: fts5) | |
15:56 | Add extra tests for corrupt database handling in fts5. (check-in: 41449f7a0b user: dan tags: fts5) | |
06:02 | Fix an fts5 build problem in main.mk. (check-in: 60045cedef user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
35 36 37 38 39 40 41 42 43 44 45 46 47 48 | #ifdef SQLITE_DEBUG # define FTS5_CORRUPT sqlite3Fts5Corrupt() int sqlite3Fts5Corrupt(void); #else # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB #endif /************************************************************************** ** Interface to code in fts5.c. */ typedef struct Fts5Global Fts5Global; int sqlite3Fts5GetTokenizer( Fts5Global*, | > > > > > > > > > > > > | 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | #ifdef SQLITE_DEBUG # define FTS5_CORRUPT sqlite3Fts5Corrupt() int sqlite3Fts5Corrupt(void); #else # define FTS5_CORRUPT SQLITE_CORRUPT_VTAB #endif /* ** The assert_nc() macro is similar to the assert() macro, except that it ** is used for assert() conditions that are true only if it can be ** guranteed that the database is not corrupt. */ #ifdef SQLITE_TEST extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) #else # define assert_nc(x) assert(x) #endif /************************************************************************** ** Interface to code in fts5.c. */ typedef struct Fts5Global Fts5Global; int sqlite3Fts5GetTokenizer( Fts5Global*, |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 | if( p->pReader ){ sqlite3_blob *pReader = p->pReader; p->pReader = 0; sqlite3_blob_close(pReader); } } static Fts5Data *fts5DataReadOrBuffer( Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid ){ Fts5Data *pRet = 0; if( p->rc==SQLITE_OK ){ int rc = SQLITE_OK; | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > < < < < < < < > > > > > > > | 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 | if( p->pReader ){ sqlite3_blob *pReader = p->pReader; p->pReader = 0; sqlite3_blob_close(pReader); } } /* ** Check if row iRowid exists in the %_data table, and that it contains ** a blob value. If so, return SQLITE_ERROR (yes - SQLITE_ERROR, not ** SQLITE_OK). If not, return SQLITE_CORRUPT_VTAB. ** ** If an error occurs (e.g. OOM or IOERR), return the relevant error code. ** ** This function does not need to be efficient. It is part of vary rarely ** invoked error handling code only. */ #if 0 static int fts5CheckMissingRowid(Fts5Index *p, i64 iRowid){ const char *zFmt = "SELECT typeof(block)=='blob' FROM '%q'.%Q WHERE id=%lld"; int bOk = 0; int rc; char *zSql; zSql = sqlite3_mprintf(zFmt, p->pConfig->zDb, p->zDataTbl, iRowid); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, &pStmt, 0); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ bOk = sqlite3_column_int(pStmt, 0); } rc = sqlite3_finalize(pStmt); } sqlite3_free(zSql); } if( rc==SQLITE_OK ){ rc = bOk ? SQLITE_ERROR : FTS5_CORRUPT; } return rc; } #endif static Fts5Data *fts5DataReadOrBuffer( Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid ){ Fts5Data *pRet = 0; if( p->rc==SQLITE_OK ){ int rc = SQLITE_OK; if( p->pReader ){ /* This call may return SQLITE_ABORT if there has been a savepoint ** rollback since it was last used. In this case a new blob handle ** is required. */ rc = sqlite3_blob_reopen(p->pReader, iRowid); if( rc==SQLITE_ABORT ){ fts5CloseReader(p); rc = SQLITE_OK; } } /* If the blob handle is not yet open, open and seek it. Otherwise, use ** the blob_reopen() API to reseek the existing blob handle. */ if( p->pReader==0 ){ Fts5Config *pConfig = p->pConfig; rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader ); } /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. ** All the reasons those functions might return SQLITE_ERROR - missing ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; if( rc==SQLITE_OK ){ u8 *aOut; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); if( pBuf ){ fts5BufferZero(pBuf); fts5BufferGrow(&rc, pBuf, nByte); |
︙ | ︙ | |||
1559 1560 1561 1562 1563 1564 1565 | ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; int iOff = pIter->iLeafOffset; /* Offset to read at */ | | < < | 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 | ** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset]; int iOff = pIter->iLeafOffset; /* Offset to read at */ pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel); } } /* ** Fts5SegIter.iLeafOffset currently points to the first byte of the ** "nSuffix" field of a term. Function parameter nKeep contains the value ** of the "nPrefix" field (if there was one - it is passed 0 if this is ** the first term in the segment). ** ** This function populates: ** ** Fts5SegIter.term ** Fts5SegIter.rowid ** ** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of ** the first position list. The position list belonging to document ** (Fts5SegIter.iRowid). */ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ |
︙ | ︙ | |||
3908 3909 3910 3911 3912 3913 3914 | } } } pIter->nEmpty = pIter->aLvl[0].s.nEmpty; pIter->bDlidx = pIter->aLvl[0].s.bDlidx; pIter->iLeaf = pIter->aLvl[0].s.iChild; | < | 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 | } } } pIter->nEmpty = pIter->aLvl[0].s.nEmpty; pIter->bDlidx = pIter->aLvl[0].s.bDlidx; pIter->iLeaf = pIter->aLvl[0].s.iChild; } static void fts5BtreeIterFree(Fts5BtreeIter *pIter){ int i; for(i=0; i<pIter->nLvl; i++){ Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; fts5NodeIterFree(&pLvl->s); |
︙ | ︙ | |||
3981 3982 3983 3984 3985 3986 3987 | int iIdx, /* Index that pSeg is a part of */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ /* Iterate through the b-tree hierarchy. */ for(fts5BtreeIterInit(p, iIdx, pSeg, &iter); | | | 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 | int iIdx, /* Index that pSeg is a part of */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ /* Iterate through the b-tree hierarchy. */ for(fts5BtreeIterInit(p, iIdx, pSeg, &iter); p->rc==SQLITE_OK && iter.bEof==0; fts5BtreeIterNext(&iter) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ int iOff; /* Offset of first term on leaf */ int i; /* Used to iterate through empty leaves */ |
︙ | ︙ |
Changes to ext/fts5/fts5_tcl.c.
︙ | ︙ | |||
18 19 20 21 22 23 24 25 26 27 28 29 30 31 | #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" #include <string.h> #include <assert.h> /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ struct SqliteDb { | > > > > > > > > | 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 | #ifdef SQLITE_ENABLE_FTS5 #include "fts5.h" #include <string.h> #include <assert.h> /* ** This variable is set to true when running corruption tests. Otherwise ** false. If it is false, extra assert() conditions in the fts5 code are ** activated - conditions that are only true if it is guaranteed that the ** fts5 database is not corrupt. */ int sqlite3_fts5_may_be_corrupt = 0; /************************************************************************* ** This is a copy of the first part of the SqliteDb structure in ** tclsqlite.c. We need it here so that the get_sqlite_pointer routine ** can extract the sqlite3* pointer from an existing Tcl SQLite ** connection. */ struct SqliteDb { |
︙ | ︙ | |||
825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 | return TCL_OK; } static void xF5tFree(ClientData clientData){ ckfree(clientData); } /* ** Entry point. */ int Fts5tcl_Init(Tcl_Interp *interp){ static struct Cmd { char *zName; Tcl_ObjCmdProc *xProc; int bTokenizeCtx; } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, | > > > > > > > > > > > > > > > > > > > > > > > > > > > | > | 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 | return TCL_OK; } static void xF5tFree(ClientData clientData){ ckfree(clientData); } /* ** sqlite3_fts5_may_be_corrupt BOOLEAN ** ** Set or clear the global "may-be-corrupt" flag. Return the old value. */ static int f5tMayBeCorrupt( void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ int bOld = sqlite3_fts5_may_be_corrupt; if( objc!=2 && objc!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "?BOOLEAN?"); return TCL_ERROR; } if( objc==2 ){ int bNew; if( Tcl_GetBooleanFromObj(interp, objv[1], &bNew) ) return TCL_ERROR; sqlite3_fts5_may_be_corrupt = bNew; } Tcl_SetObjResult(interp, Tcl_NewIntObj(bOld)); return TCL_OK; } /* ** Entry point. */ int Fts5tcl_Init(Tcl_Interp *interp){ static struct Cmd { char *zName; Tcl_ObjCmdProc *xProc; int bTokenizeCtx; } aCmd[] = { { "sqlite3_fts5_create_tokenizer", f5tCreateTokenizer, 1 }, { "sqlite3_fts5_token", f5tTokenizerReturn, 1 }, { "sqlite3_fts5_tokenize", f5tTokenize, 0 }, { "sqlite3_fts5_create_function", f5tCreateFunction, 0 }, { "sqlite3_fts5_may_be_corrupt", f5tMayBeCorrupt, 0 } }; int i; F5tTokenizerContext *pContext; pContext = ckalloc(sizeof(F5tTokenizerContext)); memset(pContext, 0, sizeof(*pContext)); |
︙ | ︙ |
Changes to ext/fts5/test/fts5corrupt.test.
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | # 2014 Dec 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); | > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 | # 2014 Dec 20 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file tests that the FTS5 'integrity-check' command detects # inconsistencies (corruption) in the on-disk backing tables. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); |
︙ | ︙ | |||
34 35 36 37 38 39 40 | set segid [lindex [fts5_level_segids t1] 0] do_test 1.3 { execsql { DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } | | | 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | set segid [lindex [fts5_level_segids t1] 0] do_test 1.3 { execsql { DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} do_test 1.4 { db_restore_and_reopen execsql { UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4); } |
︙ | ︙ |
Added ext/fts5/test/fts5corrupt2.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | # 2015 Apr 24 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file tests that FTS5 handles corrupt databases (i.e. internal # inconsistencies in the backing tables) correctly. In this case # "correctly" means without crashing. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5corrupt2 # Create a simple FTS5 table containing 100 documents. Each document # contains 10 terms, each of which start with the character "x". # expr srand(0) db func rnddoc fts5_rnddoc do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); INSERT INTO t1(t1, rank) VALUES('pgsz', 32); WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100) INSERT INTO t1 SELECT rnddoc(10) FROM ii; } set mask [expr 31 << 31] # Test 1: # # For each page in the t1_data table, open a transaction and DELETE # the t1_data entry. Then run: # # * an integrity-check, and # * unless the deleted block was a b-tree node, a query for "t1 MATCH 'x*'" # # and check that the corruption is detected in both cases. The # rollback the transaction. # # Test 2: # # Same thing, except instead of deleting a row from t1_data, replace its # blob content with integer value 14. # foreach {tno stmt} { 1 { DELETE FROM t1_data WHERE rowid=$rowid } 2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid } } { break set tn 0 foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] { incr tn #if {$tn!=224} continue do_test 1.$tno.$tn.1.$rowid { execsql { BEGIN } execsql $stmt catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} if {($rowid & $mask)==0} { # Node is a leaf node, not a b-tree node. do_catchsql_test 1.$tno.$tn.2.$rowid { SELECT rowid FROM t1 WHERE t1 MATCH 'x*' } {1 {database disk image is malformed}} } do_execsql_test 1.$tno.$tn.3.$rowid { ROLLBACK; INSERT INTO t1(t1) VALUES('integrity-check'); } {} } } # Run N-1 tests, where N is the number of bytes in the rightmost leaf page # of the fts index. For test $i, truncate the rightmost leafpage to $i # bytes. Then test both the integrity-check detects the corruption. # # Also tested is that "MATCH 'x*'" does not crash and sometimes reports # corruption. It may not report the db as corrupt because truncating the # final leaf to some sizes may create a valid leaf page. # set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}] set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}] set all [db eval {SELECT rowid FROM t1}] for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} { do_execsql_test 2.$i.1 { BEGIN; UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid; } do_catchsql_test 2.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } {1 {database disk image is malformed}} do_test 2.$i.3 { set res [catchsql {SELECT rowid FROM t1 WHERE t1 MATCH 'x*'}] expr { $res=="1 {database disk image is malformed}" || $res=="0 {$all}" } } 1 do_execsql_test 2.$i.4 { ROLLBACK; INSERT INTO t1(t1) VALUES('integrity-check'); } {} } finish_test |
Changes to ext/fts5/test/fts5rebuild.test.
︙ | ︙ | |||
35 36 37 38 39 40 41 | do_execsql_test 1.5 { DELETE FROM f1_data; } {} do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); | | | 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | do_execsql_test 1.5 { DELETE FROM f1_data; } {} do_catchsql_test 1.6 { INSERT INTO f1(f1) VALUES('integrity-check'); } {1 {database disk image is malformed}} do_execsql_test 1.7 { INSERT INTO f1(f1) VALUES('rebuild'); INSERT INTO f1(f1) VALUES('integrity-check'); } {} finish_test |
︙ | ︙ |