Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -219,11 +219,11 @@ i64 iOffset; /* Starting offset in main journal */ i64 iHdrOffset; /* See above */ Bitvec *pInSavepoint; /* Set of pages in this savepoint */ Pgno nOrig; /* Original number of pages in file */ Pgno iSubRec; /* Index of first record in sub-journal */ - u32 iFrame; /* Last frame in WAL when savepoint opened */ + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ }; /* ** A open page cache is an instance of the following structure. ** @@ -2565,11 +2565,11 @@ if( pSavepoint ){ u32 ii; /* Loop counter */ i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); if( pagerUseWal(pPager) ){ - rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->iFrame); + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); } for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ assert( offset==ii*(4+pPager->pageSize) ); rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); } @@ -5446,11 +5446,11 @@ aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage); if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } if( pagerUseWal(pPager) ){ - aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pWal); + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); } } /* Open the sub-journal, if it is not already opened. */ rc = openSubJournal(pPager); Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -212,17 +212,18 @@ ** ** The actual header in the wal-index consists of two copies of this ** object. */ struct WalIndexHdr { - u32 iChange; /* Counter incremented each transaction */ - u16 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes */ - u32 mxFrame; /* Index of last valid frame in the WAL */ - u32 nPage; /* Size of database in pages */ - u32 aSalt[2]; /* Salt-1 and salt-2 values copied from WAL header */ - u32 aCksum[2]; /* Checksum over all prior fields */ + u32 iChange; /* Counter incremented each transaction */ + u16 bigEndCksum; /* True if checksums in WAL are big-endian */ + u16 szPage; /* Database page size in bytes */ + u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 nPage; /* Size of database in pages */ + u32 aFrameCksum[2]; /* Checksum of last frame in log */ + u32 aSalt[2]; /* Two salt values copied from WAL header */ + u32 aCksum[2]; /* Checksum over all prior fields */ }; /* A block of WALINDEX_LOCK_RESERVED bytes beginning at ** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems ** only support mandatory file-locks, we do not read or write data @@ -422,18 +423,18 @@ u32 nTruncate, /* New db size (or 0 for non-commit frames) */ u8 *aData, /* Pointer to page data */ u8 *aFrame /* OUT: Write encoded frame here */ ){ int nativeCksum; /* True for native byte-order checksums */ - u32 aCksum[2]; + u32 *aCksum = pWal->hdr.aFrameCksum; assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); memcpy(&aFrame[8], pWal->hdr.aSalt, 8); nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); sqlite3Put4byte(&aFrame[16], aCksum[0]); sqlite3Put4byte(&aFrame[20], aCksum[1]); } @@ -449,12 +450,12 @@ u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* Frame data */ ){ int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; u32 pgno; /* Page number of the frame */ - u32 aCksum[2]; assert( WAL_FRAME_HDRSIZE==24 ); /* A frame is only valid if the salt values in the frame-header ** match the salt values in the wal-header. */ @@ -472,11 +473,11 @@ /* A frame is only valid if a checksum of the first 16 bytes ** of the frame-header, and the frame-data matches ** the checksum in the last 8 bytes of the frame-header. */ nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); - walChecksumBytes(nativeCksum, aFrame, 16, 0, aCksum); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) ){ /* Checksum failed. */ @@ -737,14 +738,14 @@ ** The caller must hold RECOVER lock on the wal-index file. */ static int walIndexRecover(Wal *pWal){ int rc; /* Return Code */ i64 nSize; /* Size of log file */ - WalIndexHdr hdr; /* Recovered wal-index header */ + u32 aFrameCksum[2] = {0, 0}; assert( pWal->lockState>SQLITE_SHM_READ ); - memset(&hdr, 0, sizeof(hdr)); + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); if( rc!=SQLITE_OK ){ return rc; } @@ -777,14 +778,17 @@ || szPage>SQLITE_MAX_PAGE_SIZE || szPage<512 ){ goto finished; } - hdr.bigEndCksum = pWal->hdr.bigEndCksum = (magic&0x00000001); + pWal->hdr.bigEndCksum = (magic&0x00000001); pWal->szPage = szPage; pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE, 0, pWal->hdr.aFrameCksum + ); /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc(szFrame); if( !aFrame ){ @@ -807,27 +811,28 @@ rc = walIndexAppend(pWal, ++iFrame, pgno); if( rc!=SQLITE_OK ) break; /* If nTruncate is non-zero, this is a commit record. */ if( nTruncate ){ - hdr.mxFrame = iFrame; - hdr.nPage = nTruncate; - hdr.szPage = szPage; + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = szPage; + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } } sqlite3_free(aFrame); - }else{ - memset(&hdr, 0, sizeof(hdr)); } finished: - if( rc==SQLITE_OK && hdr.mxFrame==0 ){ + if( rc==SQLITE_OK && pWal->hdr.mxFrame==0 ){ rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); } if( rc==SQLITE_OK ){ - memcpy(&pWal->hdr, &hdr, sizeof(hdr)); + pWal->hdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; walIndexWriteHdr(pWal); } return rc; } @@ -1624,29 +1629,39 @@ walIndexUnmap(pWal); } return rc; } -/* Return an integer that records the current (uncommitted) write -** position in the WAL +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current +** point in the event of a savepoint rollback (via WalSavepointUndo()). */ -u32 sqlite3WalSavepoint(Wal *pWal){ +void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ assert( pWal->lockState==SQLITE_SHM_WRITE ); - return pWal->hdr.mxFrame; + aWalData[0] = pWal->hdr.mxFrame; + aWalData[1] = pWal->hdr.aFrameCksum[0]; + aWalData[2] = pWal->hdr.aFrameCksum[1]; } -/* Move the write position of the WAL back to iFrame. Called in -** response to a ROLLBACK TO command. +/* +** Move the write position of the WAL back to the point identified by +** the values in the aWalData[] array. aWalData must point to an array +** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated +** by a call to WalSavepoint(). */ -int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){ +int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; assert( pWal->lockState==SQLITE_SHM_WRITE ); - assert( iFrame<=pWal->hdr.mxFrame ); - if( iFramehdr.mxFrame ){ + assert( aWalData[0]<=pWal->hdr.mxFrame ); + if( aWalData[0]hdr.mxFrame ){ rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); - pWal->hdr.mxFrame = iFrame; + pWal->hdr.mxFrame = aWalData[0]; + pWal->hdr.aFrameCksum[0] = aWalData[1]; + pWal->hdr.aFrameCksum[1] = aWalData[2]; if( rc==SQLITE_OK ){ walCleanupHash(pWal); walIndexUnmap(pWal); } } @@ -1692,10 +1707,11 @@ memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); if( rc!=SQLITE_OK ){ return rc; } + walChecksumBytes(1, aWalHdr, sizeof(aWalHdr), 0, pWal->hdr.aFrameCksum); } assert( pWal->szPage==szPage ); /* Write the log file. */ for(p=pList; p; p=p->pDirty){ Index: src/wal.h ================================================================== --- src/wal.h +++ src/wal.h @@ -26,16 +26,18 @@ # define sqlite3WalCloseSnapshot(z) # define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y,z) # define sqlite3WalWriteLock(y,z) 0 # define sqlite3WalUndo(x,y,z) 0 -# define sqlite3WalSavepoint(z) 0 +# define sqlite3WalSavepoint(y,z) # define sqlite3WalSavepointUndo(y,z) 0 # define sqlite3WalFrames(u,v,w,x,y,z) 0 # define sqlite3WalCheckpoint(u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 #else + +#define WAL_SAVEPOINT_NDATA 3 /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ typedef struct Wal Wal; @@ -67,15 +69,15 @@ /* Undo any frames written (but not committed) to the log */ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); /* Return an integer that records the current (uncommitted) write ** position in the WAL */ -u32 sqlite3WalSavepoint(Wal *pWal); +void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); /* Move the write position of the WAL back to iFrame. Called in ** response to a ROLLBACK TO command. */ -int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame); +int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); /* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ Index: test/wal.test ================================================================== --- test/wal.test +++ test/wal.test @@ -1346,11 +1346,12 @@ set walhdr [binary format IIIIII 931071618 3007000 $pgsz 1234 22 23] set framebody [randomblob $pgsz] set framehdr [binary format IIII $pg 5 22 23] set c1 0 set c2 0 - logcksum c1 c2 $framehdr + logcksum c1 c2 $walhdr + logcksum c1 c2 [string range $framehdr 0 7] logcksum c1 c2 $framebody set framehdr [binary format IIIIII $pg 5 22 23 $c1 $c2] set fd [open test.db-wal w] fconfigure $fd -encoding binary -translation binary Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -17,21 +17,28 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl ifcapable !wal {finish_test ; return } proc set_tvfs_hdr {file args} { + + # Set $nHdr to the number of bytes in the wal-index header: + set nHdr 80 + set nInt [expr {$nHdr/4}] + if {[llength $args]>1} { return -code error {wrong # args: should be "set_tvfs_hdr fileName ?val?"} } set blob [tvfs shm $file] if {[llength $args]} { - set blob [binary format i16a* [lindex $args 0] [string range $blob 64 end]] + set blob [ + binary format i${nInt}a* [lindex $args 0] [string range $blob $nHdr end] + ] tvfs shm $file $blob } - binary scan $blob i16 ints + binary scan $blob i${nInt} ints return $ints } proc incr_tvfs_hdr {file idx incrval} { set ints [set_tvfs_hdr $file] Index: test/walcksum.test ================================================================== --- test/walcksum.test +++ test/walcksum.test @@ -60,22 +60,17 @@ # integers ($endian must be either "big" or "little"). If the checksum looks # correct, return 1. Otherwise 0. # proc log_checksum_verify {filename iFrame endian} { set data [readfile $filename] - set c1 0 - set c2 0 - - binary scan [string range $data 8 11] I pgsz - set n [log_file_size [expr $iFrame-1] $pgsz] - binary scan [string range $data [expr $n+16] [expr $n+23]] II expect1 expect2 - log_cksum $endian c1 c2 [string range $data $n [expr $n+15]] - log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} + binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2 set expect1 [expr $expect1&0xFFFFFFFF] set expect2 [expr $expect2&0xFFFFFFFF] + expr {$c1==$expect1 && $c2==$expect2} } # # File $filename must be a WAL file on disk. Compute the checksum for frame @@ -83,27 +78,40 @@ # ($endian must be either "big" or "little"). Then write the computed # checksum into the file. # proc log_checksum_write {filename iFrame endian} { set data [readfile $filename] - set c1 0 - set c2 0 - - binary scan [string range $data 8 11] I pgsz - set n [log_file_size [expr $iFrame-1] $pgsz] - log_cksum $endian c1 c2 [string range $data $n [expr $n+15]] - log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} set bin [binary format II $c1 $c2] set fd [open $filename r+] fconfigure $fd -encoding binary fconfigure $fd -translation binary - seek $fd [expr $n+16] + seek $fd $offset puts -nonewline $fd $bin close $fd } + +proc log_checksum_calc {data iFrame endian} { + + binary scan [string range $data 8 11] I pgsz + if {$iFrame > 1} { + set n [log_file_size [expr $iFrame-2] $pgsz] + binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2 + } else { + set c1 0 + set c2 0 + log_cksum $endian c1 c2 [string range $data 0 23] + } + + set n [log_file_size [expr $iFrame-1] $pgsz] + log_cksum $endian c1 c2 [string range $data $n [expr $n+7]] + log_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] + + list [expr $n+16] $c1 $c2 +} # # File $filename must be a WAL file on disk. Set the 'magic' field of the # WAL header to indicate that checksums are $endian-endian ($endian must be # either "big" or "little"). @@ -178,18 +186,18 @@ } # Replace all checksums in the current WAL file with $endian versions. # Then check that it is still possible to recover and read the database. # + log_checksum_writemagic test2.db-wal $endian for {set f 1} {$f <= 6} {incr f} { do_test walcksum-1.$endian.3.$f { log_checksum_write test2.db-wal $f $endian log_checksum_verify test2.db-wal $f $endian } {1} } do_test walcksum-1.$endian.4.1 { - log_checksum_writemagic test2.db-wal $endian file copy -force test2.db test.db file copy -force test2.db-wal test.db-wal sqlite3 db test.db execsql { SELECT a FROM t1 } } {1 2 3 5 8 13 21} @@ -261,11 +269,11 @@ do_test walcksum-1.$endian.8.2 { log_checksum_verify test.db-wal 2 $native } {1} do_test walcksum-1.$endian.8.3 { log_checksum_verify test.db-wal 3 $native - } [expr {$native == $endian}] + } {0} do_test walcksum-1.$endian.9 { execsql { PRAGMA integrity_check; SELECT a FROM t1; @@ -273,7 +281,48 @@ } {ok 1 2 3 5 8 13 21 34 55 89} catch { db close } catch { db2 close } } + +do_test walcksum-2.1 { + file delete -force test.db test.db-wal test.db-journal + sqlite3 db test.db + execsql { + PRAGMA synchronous = NORMAL; + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA cache_size = 10; + CREATE TABLE t1(x PRIMARY KEY); + PRAGMA wal_checkpoint; + INSERT INTO t1 VALUES(randomblob(800)); + BEGIN; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 16 */ + SAVEPOINT one; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ + ROLLBACK TO one; + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ + COMMIT; + } + + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + + sqlite3 db2 test2.db + execsql { + PRAGMA integrity_check; + SELECT count(*) FROM t1; + } db2 +} {ok 256} +catch { db close } +catch { db2 close } finish_test