Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming). |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | rbu-vacuum |
Files: | files | file ages | folders |
SHA1: |
00b2f4b09ffca5156e43c4db2bfe0b2c |
User & Date: | dan 2016-04-19 16:20:24.513 |
Context
2016-04-19
| ||
17:11 | When an RBU vacuum is started on a db identified using a URI filename, pass the same URI parameters when creating the new version of the db. This ensures that RBU vacuum works with password protected databases. (check-in: ca021ba881 user: dan tags: rbu-vacuum) | |
16:20 | Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming). (check-in: 00b2f4b09f user: dan tags: rbu-vacuum) | |
2016-04-18
| ||
21:00 | Another fix to rbu vacuum for a zipvfs case. (check-in: 29407d70e4 user: dan tags: rbu-vacuum) | |
Changes
Changes to ext/rbu/rbuvacuum.test.
︙ | ︙ | |||
13 14 15 16 17 18 19 | # contains tests to ensure that the sqlite3rbu_vacuum() API works as # expected. # source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbuvacuum | | | | | | 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 | # contains tests to ensure that the sqlite3rbu_vacuum() API works as # expected. # source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbuvacuum proc do_rbu_vacuum_test {tn step} { uplevel [list do_test $tn.1 { forcedelete state.db if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } set rc [rbu step] if {$rc!="SQLITE_OK"} break if {$step==1} { rbu close } } rbu close } {SQLITE_DONE}] uplevel [list do_execsql_test $tn.2 { PRAGMA integrity_check } ok] |
︙ | ︙ | |||
45 46 47 48 49 50 51 | PRAGMA page_size = 1024; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t1 VALUES(7, 8, 9); PRAGMA integrity_check; } {ok} | | | | | | | | | | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 | PRAGMA page_size = 1024; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t1 VALUES(7, 8, 9); PRAGMA integrity_check; } {ok} do_rbu_vacuum_test 1.1 $step # A vacuum that actually reclaims space. do_execsql_test 1.2.1 { INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); PRAGMA page_count; } {12} do_execsql_test 1.2.2 { DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; PRAGMA page_count; } {12} do_rbu_vacuum_test 1.2.3 $step do_execsql_test 1.2.4 { PRAGMA page_count; } {3} # Add an index to the table. do_execsql_test 1.3.1 { CREATE INDEX t1b ON t1(b); INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); PRAGMA page_count; } {18} do_execsql_test 1.3.2 { DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; PRAGMA page_count; } {18} do_rbu_vacuum_test 1.3.3 $step do_execsql_test 1.3.4 { PRAGMA page_count; } {5} # WITHOUT ROWID table. do_execsql_test 1.4.1 { CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); DELETE FROM t2 WHERE b BETWEEN 2 AND 7; PRAGMA page_count; } {20} do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {10} # WITHOUT ROWID table with an index. do_execsql_test 1.4.1 { CREATE INDEX t2c ON t2(c); INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); DELETE FROM t2 WHERE b BETWEEN 8 AND 12; PRAGMA page_count; } {35} do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {15} do_execsql_test 1.4.4 { VACUUM; PRAGMA page_count; } {15} do_execsql_test 1.5.1 { CREATE TABLE t3(a, b, c); INSERT INTO t3 VALUES('a', 'b', 'c'); INSERT INTO t3 VALUES('d', 'e', 'f'); INSERT INTO t3 VALUES('g', 'h', 'i'); } do_rbu_vacuum_test 1.5.2 $step do_execsql_test 1.5.3 { SELECT * FROM t3 } {a b c d e f g h i} do_execsql_test 1.5.4 { CREATE INDEX t3a ON t3(a); CREATE INDEX t3b ON t3(b); CREATE INDEX t3c ON t3(c); INSERT INTO t3 VALUES('j', 'k', 'l'); DELETE FROM t3 WHERE a = 'g'; } do_rbu_vacuum_test 1.5.5 $step do_execsql_test 1.5.6 { SELECT rowid, * FROM t3 ORDER BY b } {1 a b c 2 d e f 4 j k l} do_execsql_test 1.6.1 { CREATE TABLE t4(a PRIMARY KEY, b, c); INSERT INTO t4 VALUES('a', 'b', 'c'); INSERT INTO t4 VALUES('d', 'e', 'f'); INSERT INTO t4 VALUES('g', 'h', 'i'); } do_rbu_vacuum_test 1.6.2 $step do_execsql_test 1.6.3 { SELECT * FROM t4 } {a b c d e f g h i} do_execsql_test 1.6.4 { CREATE INDEX t4a ON t4(a); CREATE INDEX t4b ON t4(b); CREATE INDEX t4c ON t4(c); INSERT INTO t4 VALUES('j', 'k', 'l'); DELETE FROM t4 WHERE a='g'; } do_rbu_vacuum_test 1.6.5 $step do_execsql_test 1.6.6 { SELECT * FROM t4 ORDER BY b } {a b c d e f j k l} } #------------------------------------------------------------------------- # Test some error cases: # # 2.1.* the db being vacuumed being in wal mode already. # 2.2.* database modified mid vacuum. # reset_db do_execsql_test 2.1.0 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); INSERT INTO t1 VALUES(5, 6); INSERT INTO t1 VALUES(7, 8); PRAGMA journal_mode = wal; INSERT INTO t1 VALUES(9, 10); } wal do_test 2.1.1 { forcedelete state.db sqlite3rbu_vacuum rbu test.db state.db rbu step } {SQLITE_ERROR} do_test 2.1.2 { list [catch { rbu close } msg] $msg } {1 {SQLITE_ERROR - cannot vacuum wal mode database}} reset_db do_execsql_test 2.2.0 { CREATE TABLE tx(a PRIMARY KEY, b BLOB); INSERT INTO tx VALUES(1, randomblob(900)); INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; } db_save_and_close for {set i 1} 1 {incr i} { forcedelete state.db db_restore_and_reopen sqlite3rbu_vacuum rbu test.db state.db for {set step 0} {$step<$i} {incr step} { rbu step } rbu close if {[file exists test.db-wal]} break execsql { INSERT INTO tx VALUES(20, 20) } do_test 2.2.$i.1 { sqlite3rbu_vacuum rbu test.db state.db rbu step } {SQLITE_BUSY} do_test 2.2.$i.2 { list [catch { rbu close } msg] $msg } {1 {SQLITE_BUSY - database modified during rbu vacuum}} } catch { db close } finish_test |
Changes to ext/rbu/sqlite3rbu.c.
︙ | ︙ | |||
395 396 397 398 399 400 401 | sqlite3_file *pReal; /* Underlying file handle */ rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ | | | 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 | sqlite3_file *pReal; /* Underlying file handle */ rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ u8 bNolock; /* True to fail EXCLUSIVE locks */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ char *zDel; /* Delete this when closing file */ const char *zWal; /* Wal filename for this main db file */ rbu_file *pWalFd; /* Wal file descriptor for this main db */ |
︙ | ︙ | |||
2369 2370 2371 2372 2373 2374 2375 | rbuFreeState(pState); } } if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); } p->eStage = 0; | | > > > | 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 | rbuFreeState(pState); } } if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); } p->eStage = 0; if( p->rc==SQLITE_OK && p->dbMain==0 ){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else if( p->pRbuFd->pWalFd ){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); |
︙ | ︙ | |||
2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 | ** Update the contents of the rbu_state table within the rbu database. The ** value stored in the RBU_STATE_STAGE column is eStage. All other values ** are determined by inspecting the rbu handle passed as the first argument. */ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; int rc; assert( p->zErrmsg==0 ); rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES " "(%d, %d), " | > | 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 | ** Update the contents of the rbu_state table within the rbu database. The ** value stored in the RBU_STATE_STAGE column is eStage. All other values ** are determined by inspecting the rbu handle passed as the first argument. */ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES " "(%d, %d), " |
︙ | ︙ | |||
3005 3006 3007 3008 3009 3010 3011 | p->zStateDb, RBU_STATE_STAGE, eStage, RBU_STATE_TBL, p->objiter.zTbl, RBU_STATE_IDX, p->objiter.zIdx, RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, | | | 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 | p->zStateDb, RBU_STATE_STAGE, eStage, RBU_STATE_TBL, p->objiter.zTbl, RBU_STATE_IDX, p->objiter.zIdx, RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)pFd->iCookie, RBU_STATE_OALSZ, p->iOalSz, RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) ); assert( pInsert==0 || rc==SQLITE_OK ); if( rc==SQLITE_OK ){ |
︙ | ︙ | |||
3427 3428 3429 3430 3431 3432 3433 | p->zErrmsg = sqlite3_mprintf("cannot update wal mode database"); }else if( p->eStage==RBU_STAGE_MOVE ){ p->eStage = RBU_STAGE_CKPT; p->nStep = 0; } } | | < | | > > | | | | | | > > > | 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 | p->zErrmsg = sqlite3_mprintf("cannot update wal mode database"); }else if( p->eStage==RBU_STAGE_MOVE ){ p->eStage = RBU_STAGE_CKPT; p->nStep = 0; } } if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) && pState->eStage!=0 ){ rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); if( pFd->iCookie!=pState->iCookie ){ /* At this point (pTargetFd->iCookie) contains the value of the ** change-counter cookie (the thing that gets incremented when a ** transaction is committed in rollback mode) currently stored on ** page 1 of the database file. */ p->rc = SQLITE_BUSY; p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", (rbuIsVacuum(p) ? "vacuum" : "update") ); } } if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; /* Open transactions both databases. The *-oal file is opened or |
︙ | ︙ | |||
3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 | rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; rbuPutU32(&aBuf[52], 0); /* largest root page number */ rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ if( iAmt>100 ){ assert( iAmt>=101 ); memset(&aBuf[101], 0, iAmt-101); rbuPutU16(&aBuf[105], iAmt & 0xFFFF); } } } #endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part ** of the definition of the SQLite file format, which may not change. */ u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; | > < < < < < < < | 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 | rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; rbuPutU32(&aBuf[52], 0); /* largest root page number */ rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ if( iAmt>100 ){ assert( iAmt>=101 ); memset(&aBuf[101], 0, iAmt-101); rbuPutU16(&aBuf[105], iAmt & 0xFFFF); } } } #endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part ** of the definition of the SQLite file format, which may not change. */ u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; } } return rc; } /* ** Write data to an rbuVfs-file. |
︙ | ︙ |