Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -2194,12 +2194,12 @@ */ static int readDbPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - i64 iOffset; /* Byte offset of file to read from */ int isInWal = 0; /* True if page is in log file */ + int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->state>=PAGER_SHARED && !MEMDB ); assert( isOpen(pPager->fd) ); if( NEVER(!isOpen(pPager->fd)) ){ @@ -2208,15 +2208,15 @@ return SQLITE_OK; } if( pagerUseWal(pPager) ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pPg->pData); + rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); } if( rc==SQLITE_OK && !isInWal ){ - iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); + i64 iOffset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; } } @@ -2830,10 +2830,19 @@ */ int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ int rc = SQLITE_OK; memset(pDest, 0, N); assert( isOpen(pPager->fd) || pPager->tempFile ); + + if( pagerUseWal(pPager) ){ + int isInWal = 0; + rc = sqlite3WalRead(pPager->pWal, 1, &isInWal, N, pDest); + if( rc!=SQLITE_OK || isInWal ){ + return rc; + } + } + if( isOpen(pPager->fd) ){ IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) rc = sqlite3OsRead(pPager->fd, pDest, N, 0); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; @@ -3053,11 +3062,12 @@ sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, pPager->fd, - (pPager->noSync ? 0 : pPager->sync_flags), pTmp + (pPager->noSync ? 0 : pPager->sync_flags), + pPager->pageSize, pTmp ); pPager->pWal = 0; #endif pager_reset(pPager); if( MEMDB ){ @@ -5831,11 +5841,12 @@ int rc = SQLITE_OK; if( pPager->pWal ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), - zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg + pPager->pageSize, zBuf, + pPager->xBusyHandler, pPager->pBusyHandlerArg ); } return rc; } @@ -5906,15 +5917,15 @@ if( rc==SQLITE_OK && pPager->pWal ){ rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), - (u8*)pPager->pTmpSpace + pPager->pageSize, (u8*)pPager->pTmpSpace ); pPager->pWal = 0; } } return rc; } #endif #endif /* SQLITE_OMIT_DISKIO */ Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -708,25 +708,32 @@ */ static int walCheckpoint( Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags for OsSync() (or 0) */ + int nBuf, /* Size of zBuf in bytes */ u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ int pgsz = pWal->hdr.pgsz; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ - if( pWal->hdr.iLastPg==0 ){ - return SQLITE_OK; - } - /* Allocate the iterator */ pIter = walIteratorInit(pWal); if( !pIter ) return SQLITE_NOMEM; + + if( pWal->hdr.iLastPg==0 ){ + rc = SQLITE_OK; + goto out; + } + + if( pWal->hdr.pgsz!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + goto out; + } /* Sync the log file to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pFd, sync_flags); if( rc!=SQLITE_OK ) goto out; @@ -786,11 +793,12 @@ */ int sqlite3WalClose( Wal *pWal, /* Wal to close */ sqlite3_file *pFd, /* Database file */ int sync_flags, /* Flags to pass to OsSync() (or 0) */ - u8 *zBuf /* Buffer of at least page-size bytes */ + int nBuf, + u8 *zBuf /* Buffer of at least nBuf bytes */ ){ int rc = SQLITE_OK; if( pWal ){ int isDelete = 0; /* True to unlink wal and wal-index files */ @@ -802,11 +810,11 @@ ** ** The EXCLUSIVE lock is not released before returning. */ rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf); if( rc==SQLITE_OK ){ isDelete = 1; } walIndexUnmap(pWal); } @@ -951,11 +959,17 @@ } /* ** Read a page from the log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ +int sqlite3WalRead( + Wal *pWal, + Pgno pgno, + int *pInWal, + int nOut, + u8 *pOut +){ u32 iRead = 0; u32 *aData; int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE ); @@ -1009,11 +1023,11 @@ ** required page. Read and return data from the log file. */ if( iRead ){ i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE; *pInWal = 1; - return sqlite3OsRead(pWal->pFd, pOut, pWal->hdr.pgsz, iOffset); + return sqlite3OsRead(pWal->pFd, pOut, nOut, iOffset); } *pInWal = 0; return SQLITE_OK; } @@ -1264,10 +1278,11 @@ */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of temporary buffer */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ){ int rc; /* Return code */ @@ -1286,11 +1301,11 @@ } /* Copy data from the log to the database file. */ rc = walIndexReadHdr(pWal, &isChanged); if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + rc = walCheckpoint(pWal, pFd, sync_flags, nBuf, zBuf); } if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was ** performed, then the pager-cache associated with log pWal is now ** out of date. So zero the cached wal-index header to ensure that Index: src/wal.h ================================================================== --- src/wal.h +++ src/wal.h @@ -22,11 +22,11 @@ #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalOpenSnapshot(y,z) 0 # define sqlite3WalCloseSnapshot(z) -# define sqlite3WalRead(w,x,y,z) 0 +# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y,z) # define sqlite3WalWriteLock(y,z) 0 # define sqlite3WalUndo(x,y,z) 0 # define sqlite3WalSavepoint(z) 0 # define sqlite3WalSavepointUndo(y,z) 0 @@ -40,11 +40,11 @@ */ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal); -int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, u8 *zBuf); +int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, int, u8 *); /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database ** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and ** preserves the current state even if the other threads or processes @@ -53,11 +53,11 @@ */ int sqlite3WalOpenSnapshot(Wal *pWal, int *); void sqlite3WalCloseSnapshot(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut); +int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); /* Return the size of the database as it existed at the beginning ** of the snapshot */ void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno); @@ -81,10 +81,11 @@ /* Copy pages from the log to the database file */ int sqlite3WalCheckpoint( Wal *pWal, /* Write-ahead log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of buffer nBuf */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ); Index: test/wal.test ================================================================== --- test/wal.test +++ test/wal.test @@ -1190,11 +1190,155 @@ db close file size test.db } [expr 512*171] } sqlite3_test_control_pending_byte $old_pending_byte + +#------------------------------------------------------------------------- +# This test - wal-18.* - verifies a couple of specific conditions that +# may be encountered while recovering a log file are handled correctly: +# +# wal-18.1.* When the first 32-bits of a frame checksum is correct but +# the second 32-bits are false, and +# +# wal-18.2.* When the page-size field that occurs at the start of a log +# file is a power of 2 greater than 16384 or smaller than 512. +# +file delete -force test.db test.db-wal test.db-journal +do_test wal-18.0 { + sqlite3 db test.db + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = OFF; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(0, 0); + PRAGMA wal_checkpoint; + + INSERT INTO t1 VALUES(1, 2); -- frames 1 and 2 + INSERT INTO t1 VALUES(3, 4); -- frames 3 and 4 + INSERT INTO t1 VALUES(5, 6); -- frames 5 and 6 + } + + file copy -force test.db testX.db + file copy -force test.db-wal testX.db-wal + db close + list [file size testX.db] [file size testX.db-wal] +} [list [expr 3*1024] [log_file_size 6 1024]] + +foreach {nFrame result} { + 0 {0 0} + 1 {0 0} + 2 {0 0 1 2} + 3 {0 0 1 2} + 4 {0 0 1 2 3 4} + 5 {0 0 1 2 3 4} + 6 {0 0 1 2 3 4 5 6} +} { + do_test wal-18.1.$nFrame { + file copy -force testX.db test.db + file copy -force testX.db-wal test.db-wal + + hexio_write test.db-wal [expr 12 + $nFrame*(16+1024) + 12] 00000000 + + sqlite3 db test.db + execsql { + SELECT * FROM t1; + PRAGMA integrity_check; + } + } [concat $result ok] + db close +} + +proc randomblob {pgsz} { + sqlite3 rbdb :memory: + set blob [rbdb one {SELECT randomblob($pgsz)}] + rbdb close + set blob +} + +proc logcksum {ckv1 ckv2 blob} { + upvar $ckv1 c1 + upvar $ckv2 c2 + + binary scan $blob iu* values + foreach v $values { + incr c1 $v + incr c2 $c1 + } + + set c1 [expr ($c1 + ($c1>>24))&0xFFFFFFFF] + set c2 [expr ($c2 + ($c2>>24))&0xFFFFFFFF] +} + +file copy -force test.db testX.db +foreach {tn pgsz works} { + 1 128 0 + 2 256 0 + 3 512 1 + 4 1024 1 + 5 2048 1 + 6 4096 1 + 7 8192 1 + 8 16384 1 + 9 32768 1 + 10 65536 0 +} { + + for {set pg 1} {$pg <= 3} {incr pg} { + file copy -force testX.db test.db + file delete -force test.db-wal + + # Check that the database now exists and consists of three pages. And + # that there is no associated wal file. + # + do_test wal-18.2.$tn.$pg.1 { file exists test.db-wal } 0 + do_test wal-18.2.$tn.$pg.2 { file exists test.db } 1 + do_test wal-18.2.$tn.$pg.3 { file size test.db } [expr 1024*3] + + do_test wal-18.2.$tn.$pg.4 { + + # Create a wal file that contains a single frame (database page + # number $pg) with the commit flag set. The frame checksum is + # correct, but the contents of the database page are corrupt. + # + # The page-size in the log file header is set to $pgsz. If the + # WAL code considers $pgsz to be a valid SQLite database file page-size, + # the database will be corrupt (because the garbage frame contents + # will be treated as valid content). If $pgsz is invalid (too small + # or too large), the db will not be corrupt as the log file will + # be ignored. + # + set c1 22 + set c2 23 + set walhdr [binary format III $pgsz $c1 $c2] + set framebody [randomblob $pgsz] + set framehdr [binary format II $pg 5] + logcksum c1 c2 $framehdr + logcksum c1 c2 $framebody + set framehdr [binary format IIII $pg 5 $c1 $c2] + set fd [open test.db-wal w] + fconfigure $fd -encoding binary -translation binary + puts -nonewline $fd $walhdr + puts -nonewline $fd $framehdr + puts -nonewline $fd $framebody + close $fd + + file size test.db-wal + } [log_file_size 1 $pgsz] + + do_test wal-18.2.$tn.$pg.5 { + sqlite3 db test.db + set rc [catch { db one {PRAGMA integrity_check} } msg] + expr { $rc!=0 || $msg!="ok" } + } $works + + db close + } +} catch { db2 close } catch { db close } finish_test - Index: test/walcrash.test ================================================================== --- test/walcrash.test +++ test/walcrash.test @@ -248,24 +248,40 @@ do_test walcrash-6.$i.4 { execsql { PRAGMA main.journal_mode } } {wal} db close } +#------------------------------------------------------------------------- +# This test case simulates a crash while checkpointing the database. Page +# 1 is one of the pages overwritten by the checkpoint. This is a special +# case because it means the content of page 1 may be damaged. SQLite will +# have to determine: +# +# (a) that the database is a WAL database, and +# (b) the database page-size +# +# based on the log file. +# for {set i 1} {$i < $REPEATS} {incr i} { file delete -force test.db test.db-wal + # Select a page-size for this test. + # + set pgsz [lindex {512 1024 2048 4096 8192 16384} [expr $i%6]] + do_test walcrash-7.$i.1 { - crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 { + crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 " + PRAGMA page_size = $pgsz; PRAGMA journal_mode = wal; BEGIN; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); COMMIT; PRAGMA wal_checkpoint; CREATE INDEX i1 ON t1(a); PRAGMA wal_checkpoint; - } + " } {1 {child process exited abnormally}} do_test walcrash-7.$i.2 { sqlite3 db test.db execsql { SELECT b FROM t1 WHERE a = 1 }