Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Enhance WAL mode so that the WAL file honors the journal size limit set by PRAGMA journal_size_limit. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA1: |
db7e500f69faf1906a84991ba67d26a1 |
User & Date: | drh 2011-05-16 21:00:27.928 |
References
2011-12-16
| ||
13:28 | • Ticket [a1fa75cbdd] Database corruption following unlucky application crash in WAL mode with "PRAGMA journal_size_limit" status still Open with 3 other changes (artifact: 20ce49f1d9 user: dan) | |
Context
2011-05-17
| ||
14:41 | Avoid including fts3_term.c in the amalgamation, as it contains test code only. (check-in: f392b7ae02 user: dan tags: trunk) | |
2011-05-16
| ||
21:00 | Enhance WAL mode so that the WAL file honors the journal size limit set by PRAGMA journal_size_limit. (check-in: db7e500f69 user: drh tags: trunk) | |
2011-05-13
| ||
18:51 | Make arrangements so that virtual table inserts into shadow tables do not modify the sqlite3_last_insert_rowid() return. Document that this is how sqlite3_last_insert_rowid() works. (check-in: e569f18b98 user: drh tags: trunk) | |
Changes
Changes to src/pager.c.
︙ | ︙ | |||
6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 | ** ** Setting the size limit to -1 means no limit is enforced. ** An attempt to set a limit smaller than -1 is a no-op. */ i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ if( iLimit>=-1 ){ pPager->journalSizeLimit = iLimit; } return pPager->journalSizeLimit; } /* ** Return a pointer to the pPager->pBackup variable. The backup module ** in backup.c maintains the content of this variable. This module | > | 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 | ** ** Setting the size limit to -1 means no limit is enforced. ** An attempt to set a limit smaller than -1 is a no-op. */ i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ if( iLimit>=-1 ){ pPager->journalSizeLimit = iLimit; sqlite3WalLimit(pPager->pWal, iLimit); } return pPager->journalSizeLimit; } /* ** Return a pointer to the pPager->pBackup variable. The backup module ** in backup.c maintains the content of this variable. This module |
︙ | ︙ | |||
6694 6695 6696 6697 6698 6699 6700 | } /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, | | > | 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 | } /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } return rc; } |
︙ | ︙ |
Changes to src/wal.c.
︙ | ︙ | |||
408 409 410 411 412 413 414 415 416 417 418 419 420 421 | ** following object. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ | > | 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 | ** following object. */ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ |
︙ | ︙ | |||
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 | ** an SQLite error code is returned and *ppWal is left unmodified. */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ assert( zWalName && zWalName[0] ); | > | 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 | ** an SQLite error code is returned and *ppWal is left unmodified. */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ i64 mxWalSize, /* Truncate WAL to this size on reset */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ assert( zWalName && zWalName[0] ); |
︙ | ︙ | |||
1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 | return SQLITE_NOMEM; } pRet->pVfs = pVfs; pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = 1; } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); sqlite3OsClose(pRet->pWalFd); sqlite3_free(pRet); }else{ *ppWal = pRet; WALTRACE(("WAL%d: opened\n", pRet)); } return rc; } /* ** Find the smallest page number out of all pages held in the WAL that ** has not been returned by any prior invocation of this method on the ** same WalIterator object. Write into *piFrame the frame index where ** that page was last written into the WAL. Write into *piPage the page ** number. | > > > > > > > > | 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 | return SQLITE_NOMEM; } pRet->pVfs = pVfs; pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = 1; } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); sqlite3OsClose(pRet->pWalFd); sqlite3_free(pRet); }else{ *ppWal = pRet; WALTRACE(("WAL%d: opened\n", pRet)); } return rc; } /* ** Change the size to which the WAL file is trucated on each reset. */ void sqlite3WalLimit(Wal *pWal, i64 iLimit){ if( pWal ) pWal->mxWalSize = iLimit; } /* ** Find the smallest page number out of all pages held in the WAL that ** has not been returned by any prior invocation of this method on the ** same WalIterator object. Write into *piFrame the frame index where ** that page was last written into the WAL. Write into *piPage the page ** number. |
︙ | ︙ | |||
2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 | ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() ** to handle if this transaction is rolled back. */ int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); aSalt[1] = salt1; walIndexWriteHdr(pWal); pInfo->nBackfill = 0; for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; | > > > > > > > > > > > > > > > > | 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 | ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() ** to handle if this transaction is rolled back. */ int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ /* Limit the size of WAL file if the journal_size_limit PRAGMA is ** set to a non-negative value. Log errors encountered ** during the truncation attempt. */ if( pWal->mxWalSize>=0 ){ i64 sz; int rx; rx = sqlite3OsFileSize(pWal->pWalFd, &sz); if( rx==SQLITE_OK && (sz > pWal->mxWalSize) ){ rx = sqlite3OsTruncate(pWal->pWalFd, pWal->mxWalSize); } if( rx ){ sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); } } pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); aSalt[1] = salt1; walIndexWriteHdr(pWal); pInfo->nBackfill = 0; for(i=1; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; |
︙ | ︙ |
Changes to src/wal.h.
︙ | ︙ | |||
17 18 19 20 21 22 23 24 25 26 27 28 29 30 | #ifndef _WAL_H_ #define _WAL_H_ #include "sqliteInt.h" #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 | > | 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 | #ifndef _WAL_H_ #define _WAL_H_ #include "sqliteInt.h" #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 |
︙ | ︙ | |||
42 43 44 45 46 47 48 | /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ | | > > > | 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ void sqlite3WalLimit(Wal*, i64); /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database ** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and ** preserves the current state even if the other threads or processes ** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the ** transaction and releases the lock. |
︙ | ︙ |
Added test/wal7.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 | # 2011 May 16 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing the PRAGMA journal_size_limit when # in WAL mode. # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !wal {finish_test ; return } # Case 1: No size limit. Journal can get large. # do_test wal7-1.0 { db close forcedelete test.db sqlite3 db test.db db eval { PRAGMA page_size=1024; PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=50; -- 50 pages CREATE TABLE t1(x, y UNIQUE); INSERT INTO t1 VALUES(1,2); INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; INSERT INTO t2 SELECT x FROM t1; } expr {[file size test.db-wal]>50*1100} } 1 do_test wal7-1.1 { db eval {PRAGMA wal_checkpoint} expr {[file size test.db-wal]>50*1100} } 1 do_test wal7-1.2 { db eval {INSERT INTO t2 VALUES('hi');} expr {[file size test.db-wal]>50*1100} } 1 # Case 2: Size limit at half the autocheckpoint size. # do_test wal7-2.0 { db close forcedelete test.db sqlite3 db test.db db eval { PRAGMA page_size=1024; PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=50; -- 50 pages PRAGMA journal_size_limit=25000; CREATE TABLE t1(x, y UNIQUE); INSERT INTO t1 VALUES(1,2); INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; INSERT INTO t2 SELECT x FROM t1; } file size test.db-wal } 25000 # Case 3: Size limit of zero. # do_test wal7-3.0 { db close forcedelete test.db sqlite3 db test.db db eval { PRAGMA page_size=1024; PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=50; -- 50 pages PRAGMA journal_size_limit=0; CREATE TABLE t1(x, y UNIQUE); INSERT INTO t1 VALUES(1,2); INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; INSERT INTO t2 SELECT x FROM t1; } set sz [file size test.db-wal] expr {$sz>0 && $sz<10000} } 1 # Case 4: Size limit set before going WAL # do_test wal7-4.0 { db close forcedelete test.db sqlite3 db test.db db eval { PRAGMA page_size=1024; PRAGMA journal_size_limit=25000; PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=50; -- 50 pages CREATE TABLE t1(x, y UNIQUE); INSERT INTO t1 VALUES(1,2); INSERT INTO t1 VALUES(zeroblob(200000),4); CREATE TABLE t2(z); DELETE FROM t1; INSERT INTO t2 SELECT x FROM t1; } set sz [file size test.db-wal] } 25000 finish_test |