# 2010 May 24 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } # Read and return the contents of file $filename. Treat the content as # binary data. # proc readfile {filename} { set fd [open $filename] fconfigure $fd -encoding binary fconfigure $fd -translation binary set data [read $fd] close $fd return $data } # # File $filename must be a WAL file on disk. Check that the checksum of frame # $iFrame in the file is correct when interpreting data as $endian-endian # integers ($endian must be either "big" or "little"). If the checksum looks # correct, return 1. Otherwise 0. # proc log_checksum_verify {filename iFrame endian} { set data [readfile $filename] foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} binary scan [string range $data $offset [expr $offset+7]] II expect1 expect2 set expect1 [expr $expect1&0xFFFFFFFF] set expect2 [expr $expect2&0xFFFFFFFF] expr {$c1==$expect1 && $c2==$expect2} } # File $filename must be a WAL file on disk. Compute the checksum for frame # $iFrame in the file by interpreting data as $endian-endian integers # ($endian must be either "big" or "little"). Then write the computed # checksum into the file. # proc log_checksum_write {filename iFrame endian} { set data [readfile $filename] foreach {offset c1 c2} [log_checksum_calc $data $iFrame $endian] {} set bin [binary format II $c1 $c2] set fd [open $filename r+] fconfigure $fd -encoding binary fconfigure $fd -translation binary seek $fd $offset puts -nonewline $fd $bin close $fd } # Calculate and return the checksum for a particular frame in a WAL. # # Arguments are: # # $data Blob containing the entire contents of a WAL. # # $iFrame Frame number within the $data WAL. Frames are numbered # starting at 1. # # $endian One of "big" or "little". # # Returns a list of three elements, as follows: # # * The byte offset of the checksum belonging to frame $iFrame in the WAL. # * The first integer in the calculated version of the checksum. # * The second integer in the calculated version of the checksum. # proc log_checksum_calc {data iFrame endian} { binary scan [string range $data 8 11] I pgsz if {$iFrame > 1} { set n [wal_file_size [expr $iFrame-2] $pgsz] binary scan [string range $data [expr $n+16] [expr $n+23]] II c1 c2 } else { set c1 0 set c2 0 wal_cksum $endian c1 c2 [string range $data 0 23] } set n [wal_file_size [expr $iFrame-1] $pgsz] wal_cksum $endian c1 c2 [string range $data $n [expr $n+7]] wal_cksum $endian c1 c2 [string range $data [expr $n+24] [expr $n+24+$pgsz-1]] list [expr $n+16] $c1 $c2 } # # File $filename must be a WAL file on disk. Set the 'magic' field of the # WAL header to indicate that checksums are $endian-endian ($endian must be # either "big" or "little"). # # Also update the wal header checksum (since the wal header contents may # have changed). # proc log_checksum_writemagic {filename endian} { set val [expr {0x377f0682 | ($endian == "big" ? 1 : 0)}] set bin [binary format I $val] set fd [open $filename r+] fconfigure $fd -encoding binary fconfigure $fd -translation binary puts -nonewline $fd $bin seek $fd 0 set blob [read $fd 24] set c1 0 set c2 0 wal_cksum $endian c1 c2 $blob seek $fd 24 puts -nonewline $fd [binary format II $c1 $c2] close $fd } #------------------------------------------------------------------------- # Test cases walcksum-1.* attempt to verify the following: # # * That both native and non-native order checksum log files can # be recovered. # # * That when appending to native or non-native checksum log files # SQLite continues to use the right kind of checksums. # # * Test point 2 when the appending process is not one that recovered # the log file. # # * Test that both native and non-native checksum log files can be # checkpointed. And that after doing so the next write to the log # file occurs using native byte-order checksums. # set native "big" if {$::tcl_platform(byteOrder) == "littleEndian"} { set native "little" } foreach endian {big little} { # Create a database. Leave some data in the log file. # do_test walcksum-1.$endian.1 { catch { db close } forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db execsql { PRAGMA page_size = 1024; PRAGMA auto_vacuum = 0; PRAGMA synchronous = NORMAL; CREATE TABLE t1(a PRIMARY KEY, b); INSERT INTO t1 VALUES(1, 'one'); INSERT INTO t1 VALUES(2, 'two'); INSERT INTO t1 VALUES(3, 'three'); INSERT INTO t1 VALUES(5, 'five'); PRAGMA journal_mode = WAL; INSERT INTO t1 VALUES(8, 'eight'); INSERT INTO t1 VALUES(13, 'thirteen'); INSERT INTO t1 VALUES(21, 'twentyone'); } forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal db close list [file size test2.db] [file size test2.db-wal] } [list [expr 1024*3] [wal_file_size 6 1024]] # Verify that the checksums are valid for all frames and that they # are calculated by interpreting data in native byte-order. # for {set f 1} {$f <= 6} {incr f} { do_test walcksum-1.$endian.2.$f { log_checksum_verify test2.db-wal $f $native } 1 } # Replace all checksums in the current WAL file with $endian versions. # Then check that it is still possible to recover and read the database. # log_checksum_writemagic test2.db-wal $endian for {set f 1} {$f <= 6} {incr f} { do_test walcksum-1.$endian.3.$f { log_checksum_write test2.db-wal $f $endian log_checksum_verify test2.db-wal $f $endian } {1} } do_test walcksum-1.$endian.4.1 { forcecopy test2.db test.db forcecopy test2.db-wal test.db-wal sqlite3 db test.db execsql { SELECT a FROM t1 } } {1 2 3 5 8 13 21} # Following recovery, any frames written to the log should use the same # endianness as the existing frames. Check that this is the case. # do_test walcksum-1.$endian.5.0 { execsql { PRAGMA synchronous = NORMAL; INSERT INTO t1 VALUES(34, 'thirtyfour'); } list [file size test.db] [file size test.db-wal] } [list [expr 1024*3] [wal_file_size 8 1024]] for {set f 1} {$f <= 8} {incr f} { do_test walcksum-1.$endian.5.$f { log_checksum_verify test.db-wal $f $endian } {1} } # Now connect a second connection to the database. Check that this one # (not the one that did recovery) also appends frames to the log using # the same endianness for checksums as the existing frames. # do_test walcksum-1.$endian.6 { sqlite3 db2 test.db execsql { PRAGMA integrity_check; SELECT a FROM t1; } db2 } {ok 1 2 3 5 8 13 21 34} do_test walcksum-1.$endian.7.0 { execsql { PRAGMA synchronous = NORMAL; INSERT INTO t1 VALUES(55, 'fiftyfive'); } db2 list [file size test.db] [file size test.db-wal] } [list [expr 1024*3] [wal_file_size 10 1024]] for {set f 1} {$f <= 10} {incr f} { do_test walcksum-1.$endian.7.$f { log_checksum_verify test.db-wal $f $endian } {1} } # Now that both the recoverer and non-recoverer have added frames to the # log file, check that it can still be recovered. # forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal do_test walcksum-1.$endian.7.11 { sqlite3 db3 test2.db execsql { PRAGMA integrity_check; SELECT a FROM t1; } db3 } {ok 1 2 3 5 8 13 21 34 55} db3 close # Run a checkpoint on the database file. Then, check that any frames written # to the start of the log use native byte-order checksums. # do_test walcksum-1.$endian.8.1 { execsql { PRAGMA wal_checkpoint; INSERT INTO t1 VALUES(89, 'eightynine'); } log_checksum_verify test.db-wal 1 $native } {1} do_test walcksum-1.$endian.8.2 { log_checksum_verify test.db-wal 2 $native } {1} do_test walcksum-1.$endian.8.3 { log_checksum_verify test.db-wal 3 $native } {0} do_test walcksum-1.$endian.9 { execsql { PRAGMA integrity_check; SELECT a FROM t1; } db2 } {ok 1 2 3 5 8 13 21 34 55 89} catch { db close } catch { db2 close } } #------------------------------------------------------------------------- # Test case walcksum-2.* tests that if a statement transaction is rolled # back after frames are written to the WAL, and then (after writing some # more) the outer transaction is committed, the WAL file is still correctly # formatted (and can be recovered by a second process if required). # do_test walcksum-2.1 { forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db execsql { PRAGMA synchronous = NORMAL; PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; PRAGMA cache_size = 10; CREATE TABLE t1(x PRIMARY KEY); PRAGMA wal_checkpoint; INSERT INTO t1 VALUES(randomblob(800)); BEGIN; INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 2 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 4 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 8 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 16 */ SAVEPOINT one; INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ ROLLBACK TO one; INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 32 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 64 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 128 */ INSERT INTO t1 SELECT randomblob(800) FROM t1; /* 256 */ COMMIT; } forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal sqlite3 db2 test2.db execsql { PRAGMA integrity_check; SELECT count(*) FROM t1; } db2 } {ok 256} catch { db close } catch { db2 close } #------------------------------------------------------------------------- # Test case walcksum-3.* tests that the checksum calculation detects single # byte changes to frame or frame-header data and considers the frame # invalid as a result. # do_test walcksum-3.1 { forcedelete test.db test.db-wal test.db-journal sqlite3 db test.db execsql { PRAGMA synchronous = NORMAL; PRAGMA page_size = 1024; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, randomblob(300)); INSERT INTO t1 VALUES(2, randomblob(300)); PRAGMA journal_mode = WAL; INSERT INTO t1 VALUES(3, randomblob(300)); } file size test.db-wal } [wal_file_size 1 1024] do_test walcksum-3.2 { forcecopy test.db-wal test2.db-wal forcecopy test.db test2.db sqlite3 db2 test2.db execsql { SELECT a FROM t1 } db2 } {1 2 3} db2 close forcecopy test.db test2.db foreach incr {1 2 3 20 40 60 80 100 120 140 160 180 200 220 240 253 254 255} { do_test walcksum-3.3.$incr { set FAIL 0 for {set iOff 0} {$iOff < [wal_file_size 1 1024]} {incr iOff} { forcecopy test.db-wal test2.db-wal set fd [open test2.db-wal r+] fconfigure $fd -encoding binary fconfigure $fd -translation binary seek $fd $iOff binary scan [read $fd 1] c x seek $fd $iOff puts -nonewline $fd [binary format c [expr {($x+$incr)&0xFF}]] close $fd sqlite3 db2 test2.db if { [execsql { SELECT a FROM t1 } db2] != "1 2" } {set FAIL 1} db2 close } set FAIL } {0} } finish_test