# 2005 November 30 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # This file contains tests to ensure that the library handles malloc() failures # correctly. The emphasis of these tests are the _prepare(), _step() and # _finalize() calls. # # $Id: malloc3.test,v 1.24 2008/10/14 15:54:08 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl # Only run these tests if memory debugging is turned on. # if {!$MEMDEBUG} { puts "Skipping malloc3 tests: not compiled with -DSQLITE_MEMDEBUG..." finish_test return } #-------------------------------------------------------------------------- # NOTES ON RECOVERING FROM A MALLOC FAILURE # # The tests in this file test the behaviours described in the following # paragraphs. These tests test the behaviour of the system when malloc() fails # inside of a call to _prepare(), _step(), _finalize() or _reset(). The # handling of malloc() failures within ancillary procedures is tested # elsewhere. # # Overview: # # Executing a statement is done in three stages (prepare, step and finalize). A # malloc() failure may occur within any stage. If a memory allocation fails # during statement preparation, no statement handle is returned. From the users # point of view the system state is as if _prepare() had never been called. # # If the memory allocation fails during the _step() or _finalize() calls, then # the database may be left in one of two states (after finalize() has been # called): # # * As if the neither _step() nor _finalize() had ever been called on # the statement handle (i.e. any changes made by the statement are # rolled back). # * The current transaction may be rolled back. In this case a hot-journal # may or may not actually be present in the filesystem. # # The caller can tell the difference between these two scenarios by invoking # _get_autocommit(). # # # Handling of sqlite3_reset(): # # If a malloc() fails while executing an sqlite3_reset() call, this is handled # in the same way as a failure within _finalize(). The statement handle # is not deleted and must be passed to _finalize() for resource deallocation. # Attempting to _step() or _reset() the statement after a failed _reset() will # always return SQLITE_NOMEM. # # # Other active SQL statements: # # The effect of a malloc failure on concurrently executing SQL statements, # particularly when the statement is executing with READ_UNCOMMITTED set and # the malloc() failure mandates statement rollback only. Currently, if # transaction rollback is required, all other vdbe's are aborted. # # Non-transient mallocs in btree.c: # * The Btree structure itself # * Each BtCursor structure # # Mallocs in pager.c: # readMasterJournal() - Space to read the master journal name # pager_delmaster() - Space for the entire master journal file # # sqlite3pager_open() - The pager structure itself # sqlite3_pagerget() - Space for a new page # pager_open_journal() - Pager.aInJournal[] bitmap # sqlite3pager_write() - For in-memory databases only: history page and # statement history page. # pager_stmt_begin() - Pager.aInStmt[] bitmap # # None of the above are a huge problem. The most troublesome failures are the # transient malloc() calls in btree.c, which can occur during the tree-balance # operation. This means the tree being balanced will be internally inconsistent # after the malloc() fails. To avoid the corrupt tree being read by a # READ_UNCOMMITTED query, we have to make sure the transaction or statement # rollback occurs before sqlite3_step() returns, not during a subsequent # sqlite3_finalize(). #-------------------------------------------------------------------------- #-------------------------------------------------------------------------- # NOTES ON TEST IMPLEMENTATION # # The tests in this file are implemented differently from those in other # files. Instead, tests are specified using three primitives: SQL, PREP and # TEST. Each primitive has a single argument. Primitives are processed in # the order they are specified in the file. # # A TEST primitive specifies a TCL script as its argument. When a TEST # directive is encountered the Tcl script is evaluated. Usually, this Tcl # script contains one or more calls to [do_test]. # # A PREP primitive specifies an SQL script as its argument. When a PREP # directive is encountered the SQL is evaluated using database connection # [db]. # # The SQL primitives are where the action happens. An SQL primitive must # contain a single, valid SQL statement as its argument. When an SQL # primitive is encountered, it is evaluated one or more times to test the # behaviour of the system when malloc() fails during preparation or # execution of said statement. The Nth time the statement is executed, # the Nth malloc is said to fail. The statement is executed until it # succeeds, i.e. (M+1) times, where M is the number of mallocs() required # to prepare and execute the statement. # # Each time an SQL statement fails, the driver program (see proc [run_test] # below) figures out if a transaction has been automatically rolled back. # If not, it executes any TEST block immediately proceeding the SQL # statement, then reexecutes the SQL statement with the next value of N. # # If a transaction has been automatically rolled back, then the driver # program executes all the SQL specified as part of SQL or PREP primitives # between the current SQL statement and the most recent "BEGIN". Any # TEST block immediately proceeding the SQL statement is evaluated, and # then the SQL statement reexecuted with the incremented N value. # # That make any sense? If not, read the code in [run_test] and it might. # # Extra restriction imposed by the implementation: # # * If a PREP block starts a transaction, it must finish it. # * A PREP block may not close a transaction it did not start. # #-------------------------------------------------------------------------- # These procs are used to build up a "program" in global variable # ::run_test_script. At the end of this file, the proc [run_test] is used # to execute the program (and all test cases contained therein). # set ::run_test_script [list] proc TEST {id t} {lappend ::run_test_script -test [list $id $t]} proc PREP {p} {lappend ::run_test_script -prep [string trim $p]} proc DEBUG {s} {lappend ::run_test_script -debug $s} # SQL -- # # SQL ?-norollback? # # Add an 'SQL' primitive to the program (see notes above). If the -norollback # switch is present, then the statement is not allowed to automatically roll # back any active transaction if malloc() fails. It must rollback the statement # transaction only. # proc SQL {a1 {a2 ""}} { # An SQL primitive parameter is a list of two elements, a boolean value # indicating if the statement may cause transaction rollback when malloc() # fails, and the sql statement itself. if {$a2 == ""} { lappend ::run_test_script -sql [list true [string trim $a1]] } else { lappend ::run_test_script -sql [list false [string trim $a2]] } } # TEST_AUTOCOMMIT -- # # A shorthand test to see if a transaction is active or not. The first # argument - $id - is the integer number of the test case. The second # argument is either 1 or 0, the expected value of the auto-commit flag. # proc TEST_AUTOCOMMIT {id a} { TEST $id "do_test \$testid { sqlite3_get_autocommit \$::DB } {$a}" } #-------------------------------------------------------------------------- # Start of test program declaration # # Warm body test. A malloc() fails in the middle of a CREATE TABLE statement # in a single-statement transaction on an empty database. Not too much can go # wrong here. # TEST 1 { do_test $testid { execsql {SELECT tbl_name FROM sqlite_master;} } {} } SQL { CREATE TABLE IF NOT EXISTS abc(a, b, c); } TEST 2 { do_test $testid.1 { execsql {SELECT tbl_name FROM sqlite_master;} } {abc} } # Insert a couple of rows into the table. each insert is in its own # transaction. test that the table is unpopulated before running the inserts # (and hence after each failure of the first insert), and that it has been # populated correctly after the final insert succeeds. # TEST 3 { do_test $testid.2 { execsql {SELECT * FROM abc} } {} } SQL {INSERT INTO abc VALUES(1, 2, 3);} SQL {INSERT INTO abc VALUES(4, 5, 6);} SQL {INSERT INTO abc VALUES(7, 8, 9);} TEST 4 { do_test $testid { execsql {SELECT * FROM abc} } {1 2 3 4 5 6 7 8 9} } # Test a CREATE INDEX statement. Because the table 'abc' is so small, the index # will all fit on a single page, so this doesn't test too much that the CREATE # TABLE statement didn't test. A few of the transient malloc()s in btree.c # perhaps. # SQL {CREATE INDEX abc_i ON abc(a, b, c);} TEST 4 { do_test $testid { execsql { SELECT * FROM abc ORDER BY a DESC; } } {7 8 9 4 5 6 1 2 3} } # Test a DELETE statement. Also create a trigger and a view, just to make sure # these statements don't have any obvious malloc() related bugs in them. Note # that the test above will be executed each time the DELETE fails, so we're # also testing rollback of a DELETE from a table with an index on it. # SQL {DELETE FROM abc WHERE a > 2;} SQL {CREATE TRIGGER abc_t AFTER INSERT ON abc BEGIN SELECT 'trigger!'; END;} SQL {CREATE VIEW abc_v AS SELECT * FROM abc;} TEST 5 { do_test $testid { execsql { SELECT name, tbl_name FROM sqlite_master ORDER BY name; SELECT * FROM abc; } } {abc abc abc_i abc abc_t abc abc_v abc_v 1 2 3} } set sql { BEGIN;DELETE FROM abc; } for {set i 1} {$i < 15} {incr i} { set a $i set b "String value $i" set c [string repeat X $i] append sql "INSERT INTO abc VALUES ($a, '$b', '$c');" } append sql {COMMIT;} PREP $sql SQL { DELETE FROM abc WHERE oid IN (SELECT oid FROM abc ORDER BY random() LIMIT 5); } TEST 6 { do_test $testid.1 { execsql {SELECT count(*) FROM abc} } {94} do_test $testid.2 { execsql { SELECT min( (oid == a) AND 'String value ' || a == b AND a == length(c) ) FROM abc; } } {1} } SQL { DELETE FROM abc WHERE oid IN (SELECT oid FROM abc ORDER BY random() LIMIT 5); } TEST 7 { do_test $testid { execsql {SELECT count(*) FROM abc} } {89} do_test $testid { execsql { SELECT min( (oid == a) AND 'String value ' || a == b AND a == length(c) ) FROM abc; } } {1} } SQL { DELETE FROM abc WHERE oid IN (SELECT oid FROM abc ORDER BY random() LIMIT 5); } TEST 9 { do_test $testid { execsql {SELECT count(*) FROM abc} } {84} do_test $testid { execsql { SELECT min( (oid == a) AND 'String value ' || a == b AND a == length(c) ) FROM abc; } } {1} } set padding [string repeat X 500] PREP [subst { DROP TABLE abc; CREATE TABLE abc(a PRIMARY KEY, padding, b, c); INSERT INTO abc VALUES(0, '$padding', 2, 2); INSERT INTO abc VALUES(3, '$padding', 5, 5); INSERT INTO abc VALUES(6, '$padding', 8, 8); }] TEST 10 { do_test $testid { execsql {SELECT a, b, c FROM abc} } {0 2 2 3 5 5 6 8 8} } SQL {BEGIN;} SQL {INSERT INTO abc VALUES(9, 'XXXXX', 11, 12);} TEST_AUTOCOMMIT 11 0 SQL -norollback {UPDATE abc SET a = a + 1, c = c + 1;} TEST_AUTOCOMMIT 12 0 SQL {DELETE FROM abc WHERE a = 10;} TEST_AUTOCOMMIT 13 0 SQL {COMMIT;} TEST 14 { do_test $testid.1 { sqlite3_get_autocommit $::DB } {1} do_test $testid.2 { execsql {SELECT a, b, c FROM abc} } {1 2 3 4 5 6 7 8 9} } PREP [subst { DROP TABLE abc; CREATE TABLE abc(a, padding, b, c); INSERT INTO abc VALUES(1, '$padding', 2, 3); INSERT INTO abc VALUES(4, '$padding', 5, 6); INSERT INTO abc VALUES(7, '$padding', 8, 9); CREATE INDEX abc_i ON abc(a, padding, b, c); }] TEST 15 { db eval {PRAGMA cache_size = 10} } SQL {BEGIN;} SQL -norllbck {INSERT INTO abc (oid, a, padding, b, c) SELECT NULL, * FROM abc} TEST 16 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 2 4 2 7 2} } SQL -norllbck {INSERT INTO abc (oid, a, padding, b, c) SELECT NULL, * FROM abc} TEST 17 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 4 4 4 7 4} } SQL -norllbck {INSERT INTO abc (oid, a, padding, b, c) SELECT NULL, * FROM abc} TEST 18 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 8 4 8 7 8} } SQL -norllbck {INSERT INTO abc (oid, a, padding, b, c) SELECT NULL, * FROM abc} TEST 19 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 16 4 16 7 16} } SQL {COMMIT;} TEST 21 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 16 4 16 7 16} } SQL {BEGIN;} SQL {DELETE FROM abc WHERE oid %2} TEST 22 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 8 4 8 7 8} } SQL {DELETE FROM abc} TEST 23 { do_test $testid { execsql {SELECT * FROM abc} } {} } SQL {ROLLBACK;} TEST 24 { do_test $testid { execsql {SELECT a, count(*) FROM abc GROUP BY a;} } {1 16 4 16 7 16} } # Test some schema modifications inside of a transaction. These should all # cause transaction rollback if they fail. Also query a view, to cover a bit # more code. # PREP {DROP VIEW abc_v;} TEST 25 { do_test $testid { execsql { SELECT name, tbl_name FROM sqlite_master; } } {abc abc abc_i abc} } SQL {BEGIN;} SQL {CREATE TABLE def(d, e, f);} SQL {CREATE TABLE ghi(g, h, i);} TEST 26 { do_test $testid { execsql { SELECT name, tbl_name FROM sqlite_master; } } {abc abc abc_i abc def def ghi ghi} } SQL {CREATE VIEW v1 AS SELECT * FROM def, ghi} SQL {CREATE UNIQUE INDEX ghi_i1 ON ghi(g);} TEST 27 { do_test $testid { execsql { SELECT name, tbl_name FROM sqlite_master; } } {abc abc abc_i abc def def ghi ghi v1 v1 ghi_i1 ghi} } SQL {INSERT INTO def VALUES('a', 'b', 'c')} SQL {INSERT INTO def VALUES(1, 2, 3)} SQL -norollback {INSERT INTO ghi SELECT * FROM def} TEST 28 { do_test $testid { execsql { SELECT * FROM def, ghi WHERE d = g; } } {a b c a b c 1 2 3 1 2 3} } SQL {COMMIT} TEST 29 { do_test $testid { execsql { SELECT * FROM v1 WHERE d = g; } } {a b c a b c 1 2 3 1 2 3} } # Test a simple multi-file transaction # file delete -force test2.db ifcapable attach { SQL {ATTACH 'test2.db' AS aux;} SQL {BEGIN} SQL {CREATE TABLE aux.tbl2(x, y, z)} SQL {INSERT INTO tbl2 VALUES(1, 2, 3)} SQL {INSERT INTO def VALUES(4, 5, 6)} TEST 30 { do_test $testid { execsql { SELECT * FROM tbl2, def WHERE d = x; } } {1 2 3 1 2 3} } SQL {COMMIT} TEST 31 { do_test $testid { execsql { SELECT * FROM tbl2, def WHERE d = x; } } {1 2 3 1 2 3} } } # Test what happens when a malloc() fails while there are other active # statements. This changes the way sqlite3VdbeHalt() works. TEST 32 { if {![info exists ::STMT32]} { set sql "SELECT name FROM sqlite_master" set ::STMT32 [sqlite3_prepare $::DB $sql -1 DUMMY] do_test $testid { sqlite3_step $::STMT32 } {SQLITE_ROW} } } SQL BEGIN TEST 33 { do_test $testid { execsql {SELECT * FROM ghi} } {a b c 1 2 3} } SQL -norollback { -- There is a unique index on ghi(g), so this statement may not cause -- an automatic ROLLBACK. Hence the "-norollback" switch. INSERT INTO ghi SELECT '2'||g, h, i FROM ghi; } TEST 34 { if {[info exists ::STMT32]} { do_test $testid { sqlite3_finalize $::STMT32 } {SQLITE_OK} unset ::STMT32 } } SQL COMMIT # # End of test program declaration #-------------------------------------------------------------------------- proc run_test {arglist iRepeat {pcstart 0} {iFailStart 1}} { if {[llength $arglist] %2} { error "Uneven number of arguments to TEST" } for {set i 0} {$i < $pcstart} {incr i} { set k2 [lindex $arglist [expr 2 * $i]] set v2 [lindex $arglist [expr 2 * $i + 1]] set ac [sqlite3_get_autocommit $::DB] ;# Auto-Commit switch -- $k2 { -sql {db eval [lindex $v2 1]} -prep {db eval $v2} } set nac [sqlite3_get_autocommit $::DB] ;# New Auto-Commit if {$ac && !$nac} {set begin_pc $i} } db rollback_hook [list incr ::rollback_hook_count] set iFail $iFailStart set pc $pcstart while {$pc*2 < [llength $arglist]} { # Id of this iteration: set k [lindex $arglist [expr 2 * $pc]] set iterid "pc=$pc.iFail=$iFail$k" set v [lindex $arglist [expr 2 * $pc + 1]] switch -- $k { -test { foreach {id script} $v {} incr pc } -sql { set ::rollback_hook_count 0 set ac [sqlite3_get_autocommit $::DB] ;# Auto-Commit sqlite3_memdebug_fail $iFail -repeat 0 set rc [catch {db eval [lindex $v 1]} msg] ;# True error occurs set nac [sqlite3_get_autocommit $::DB] ;# New Auto-Commit if {$rc != 0 && $nac && !$ac} { # Before [db eval] the auto-commit flag was clear. Now it # is set. Since an error occured we assume this was not a # commit - therefore a rollback occured. Check that the # rollback-hook was invoked. do_test malloc3-rollback_hook.$iterid { set ::rollback_hook_count } {1} } set nFail [sqlite3_memdebug_fail -1 -benigncnt nBenign] if {$rc == 0} { # Successful execution of sql. The number of failed malloc() # calls should be equal to the number of benign failures. # Otherwise a malloc() failed and the error was not reported. # if {$nFail!=$nBenign} { error "Unreported malloc() failure" } if {$ac && !$nac} { # Before the [db eval] the auto-commit flag was set, now it # is clear. We can deduce that a "BEGIN" statement has just # been successfully executed. set begin_pc $pc } incr pc set iFail 1 integrity_check "malloc3-(integrity).$iterid" } elseif {[regexp {.*out of memory} $msg] || [db errorcode] == 3082} { # Out of memory error, as expected. # integrity_check "malloc3-(integrity).$iterid" incr iFail if {$nac && !$ac} { if {![lindex $v 0] && [db errorcode] != 3082} { # error "Statement \"[lindex $v 1]\" caused a rollback" } for {set i $begin_pc} {$i < $pc} {incr i} { set k2 [lindex $arglist [expr 2 * $i]] set v2 [lindex $arglist [expr 2 * $i + 1]] set catchupsql "" switch -- $k2 { -sql {set catchupsql [lindex $v2 1]} -prep {set catchupsql $v2} } db eval $catchupsql } } } else { error $msg } while {[lindex $arglist [expr 2 * ($pc -1)]] == "-test"} { incr pc -1 } } -prep { db eval $v incr pc } -debug { eval $v incr pc } default { error "Unknown switch: $k" } } } } # Turn of the Tcl interface's prepared statement caching facility. Then # run the tests with "persistent" malloc failures. sqlite3_extended_result_codes db 1 db cache size 0 run_test $::run_test_script 1 # Close and reopen the db. db close file delete -force test.db test.db-journal test2.db test2.db-journal sqlite3 db test.db sqlite3_extended_result_codes db 1 set ::DB [sqlite3_connection_pointer db] # Turn off the Tcl interface's prepared statement caching facility in # the new connnection. Then run the tests with "transient" malloc failures. db cache size 0 run_test $::run_test_script 0 sqlite3_memdebug_fail -1 finish_test