Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -737,10 +737,40 @@ } sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } + +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} /* ** Configuration settings for an individual database connection */ int sqlite3_db_config(sqlite3 *db, int op, ...){ Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -4471,10 +4471,29 @@ } return pager_error(pPager, rc); } +/* +** Flush all unreferenced dirty pages to disk. +*/ +int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} /* ** Allocate and initialize a new Pager object and put a pointer to it ** in *ppPager. The pager should eventually be freed by passing it ** to sqlite3PagerClose(). @@ -5894,13 +5913,14 @@ */ int sqlite3PagerWrite(PgHdr *pPg){ Pager *pPager = pPg->pPager; assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->errCode ){ + return pPager->errCode; + }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ return pagerWriteLargeSector(pPg); }else{ @@ -6074,18 +6094,21 @@ ** successful, or the connection is in WAL mode, SQLITE_OK is returned. ** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ int sqlite3PagerExclusiveLock(Pager *pPager){ - int rc = SQLITE_OK; - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED - ); + int rc = pPager->errCode; assert( assert_pager_state(pPager) ); - if( 0==pagerUseWal(pPager) ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } } return rc; } /* Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -130,10 +130,11 @@ int sqlite3PagerSetJournalMode(Pager *, int); int sqlite3PagerGetJournalMode(Pager*); int sqlite3PagerOkToChangeJournalMode(Pager*); i64 sqlite3PagerJournalSizeLimit(Pager *, i64); sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +int sqlite3PagerFlush(Pager*); /* Functions used to obtain and release page references. */ int sqlite3PagerAcquire(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); #define sqlite3PagerGet(A,B,C) sqlite3PagerAcquire(A,B,C,0) DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -7790,10 +7790,39 @@ ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +/* +** CAPI3REF: Flush caches to disk mid-transaction +** +** If a write-transaction is open when this function is called, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). Dirty pages are flushed for all +** databases - "main", "temp" and any attached databases. +** +** If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. If said locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite error code returned to the caller immediately. +** +** Otherwise, if no error occurs, SQLITE_OK is returned. +** +** This function does not set the database handle error code or message +** returned by the sqlite3_errcode() and sqlite3_errmsg() functions. +*/ +int sqlite3_db_cacheflush(sqlite3*); /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. */ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -4685,10 +4685,38 @@ if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_db_release_memory(db); Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_OK; } + +/* +** Usage: sqlite3_db_cacheflush DB +** +** Attempt to flush any dirty pages to disk. +*/ +static int test_db_cacheflush( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + rc = sqlite3_db_cacheflush(db); + if( rc ){ + Tcl_SetResult(interp, (char *)sqlite3ErrStr(rc), TCL_STATIC); + return TCL_ERROR; + } + + Tcl_ResetResult(interp); + return TCL_OK; +} /* ** Usage: sqlite3_db_filename DB DBNAME ** ** Return the name of a file associated with a database. @@ -6874,10 +6902,11 @@ { "sqlite3_stmt_busy", test_stmt_busy ,0 }, { "uses_stmt_journal", uses_stmt_journal ,0 }, { "sqlite3_release_memory", test_release_memory, 0}, { "sqlite3_db_release_memory", test_db_release_memory, 0}, + { "sqlite3_db_cacheflush", test_db_cacheflush, 0}, { "sqlite3_db_filename", test_db_filename, 0}, { "sqlite3_db_readonly", test_db_readonly, 0}, { "sqlite3_soft_heap_limit", test_soft_heap_limit, 0}, { "sqlite3_thread_cleanup", test_thread_cleanup, 0}, { "sqlite3_pager_refcounts", test_pager_refcounts, 0}, ADDED test/cacheflush.test Index: test/cacheflush.test ================================================================== --- /dev/null +++ test/cacheflush.test @@ -0,0 +1,324 @@ +# 2011 November 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for sqlite3_db_cacheflush API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cacheflush +test_set_config_pagecache 0 0 + +# Run the supplied SQL on a copy of the database currently stored on +# disk in file $dbfile. +proc diskquery {dbfile sql} { + forcecopy $dbfile dq.db + sqlite3 dq dq.db + set res [execsql $sql dq] + dq close + set res +} + +# Simplest possible test. +# +do_execsql_test 1.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + BEGIN; + INSERT INTO t1 VALUES(3, 4); +} +do_test 1.1.1 { + diskquery test.db { SELECT * FROM t1 } +} {1 2} +do_test 1.1.2 { + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1 } +} {1 2 3 4} + +# Test that multiple pages may be flushed to disk. +# +do_execsql_test 1.2.0 { + COMMIT; + CREATE TABLE t2(a, b); + BEGIN; + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t2 VALUES('a', 'b'); +} +do_test 1.2.1 { + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4} +do_test 1.2.2 { + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + } +} {1 2 3 4 5 6 a b} + +# Test that pages with nRef!=0 are not flushed to disk. +# +do_execsql_test 1.3.0 { + COMMIT; + CREATE TABLE t3(a, b); + BEGIN; + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t3 VALUES('i', 'ii'); +} +do_test 1.3.1 { + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 a b} +do_test 1.3.2 { + db eval { SELECT a FROM t1 } { + if {$a==3} { + sqlite3_db_cacheflush db + } + } + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 a b c d i ii} +do_test 1.3.2 { + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } +} {1 2 3 4 5 6 7 8 a b c d i ii} + +# Check that SQLITE_BUSY is returned if pages cannot be flushed due to +# conflicting read locks. +# +do_execsql_test 1.4.0 { + COMMIT; + BEGIN; + INSERT INTO t1 VALUES(9, 10); +} +do_test 1.4.1 { + sqlite3 db2 test.db + db2 eval { + BEGIN; + SELECT * FROM t1; + } + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8} +do_test 1.4.2 { + list [catch { sqlite3_db_cacheflush db } msg] $msg +} {1 {database is locked}} +do_test 1.4.3 { + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8} +do_test 1.4.4 { + db2 close + sqlite3_db_cacheflush db + diskquery test.db { + SELECT * FROM t1; + } +} {1 2 3 4 5 6 7 8 9 10} +do_execsql_test 1.4.5 { COMMIT } + +#------------------------------------------------------------------------- +# Test that ATTACHed database caches are also flushed. +# +forcedelete test.db2 +do_execsql_test 2.1.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t4(x, y); + INSERT INTO t4 VALUES('A', 'B'); + BEGIN; + INSERT INTO t1 VALUES(11, 12); + INSERT INTO t4 VALUES('C', 'D'); +} +do_test 2.1.1 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10} +do_test 2.1.2 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B} +do_test 2.1.3 { + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.1.4 { + sqlite3_db_cacheflush db + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D} +do_execsql_test 2.1.5 { COMMIT } + +# And that hitting an SQLITE_BUSY when flushing "main" does not stop +# SQLite from going on to flush "aux". +# +do_execsql_test 2.2.0 { + BEGIN; + INSERT INTO t1 VALUES(13, 14); + INSERT INTO t4 VALUES('E', 'F'); +} +do_test 2.2.1 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.2.2 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D} +do_test 2.2.3 { + sqlite3 db2 test.db + execsql { + BEGIN; + SELECT * FROM t1; + } db2 + list [catch { sqlite3_db_cacheflush db } msg] $msg +} {1 {database is locked}} +do_test 2.2.4 { + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12} +do_test 2.2.5 { + diskquery test.db2 { SELECT * FROM t4; } +} {A B C D E F} +do_test 2.2.6 { + db2 close + sqlite3_db_cacheflush db + diskquery test.db { SELECT * FROM t1; } +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} +do_execsql_test 2.2.7 { COMMIT } + +#------------------------------------------------------------------------- +# Test that nothing terrible happens if sqlite3_db_cacheflush() is +# called on an in-memory database. +# +do_test 3.0 { + db close + sqlite3 db :memory: + db eval { + CREATE TABLE t1(x PRIMARY KEY); + CREATE TABLE t2(y PRIMARY KEY); + BEGIN; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t2 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t2 VALUES(randomblob(100)); + } + sqlite3_db_cacheflush db +} {} + +do_execsql_test 3.1 { PRAGMA integrity_check } ok +do_execsql_test 3.2 { COMMIT } +do_execsql_test 3.3 { PRAGMA integrity_check } ok +do_execsql_test 3.4 { + SELECT count(*) FROM t1; + SELECT count(*) FROM t2; +} {2 2} + +#------------------------------------------------------------------------- +# Test that calling sqlite3_db_cacheflush() does not interfere with +# savepoint transactions. +# +do_test 4.0 { + reset_db + execsql { + CREATE TABLE ta(a, aa); + CREATE TABLE tb(b, bb); + INSERT INTO ta VALUES('a', randomblob(500)); + INSERT INTO tb VALUES('b', randomblob(500)); + BEGIN; + UPDATE ta SET a = 'A'; + SAVEPOINT one; + UPDATE tb SET b = 'B'; + } + + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A B} + +do_test 4.1 { + execsql { + ROLLBACK TO one; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b} + +do_test 4.2 { + execsql { + INSERT INTO tb VALUES('c', randomblob(10)); + INSERT INTO tb VALUES('d', randomblob(10)); + INSERT INTO tb VALUES('e', randomblob(10)); + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b c d e} + +do_test 4.3 { + execsql { + SAVEPOINT two; + UPDATE tb SET b = upper(b); + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A B C D E} + +do_test 4.4 { + execsql { + ROLLBACK TO two; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b c d e} + +do_test 4.4 { + execsql { + ROLLBACK TO one; + } + sqlite3_db_cacheflush db + diskquery test.db { + SELECT a FROM ta; + SELECT b FROM tb; + } +} {A b} + +do_test 4.5 { + execsql { + ROLLBACK; + SELECT a FROM ta; + SELECT b FROM tb; + } +} {a b} + +test_restore_config_pagecache +finish_test + ADDED test/cffault.test Index: test/cffault.test ================================================================== --- /dev/null +++ test/cffault.test @@ -0,0 +1,159 @@ +# 2011 November 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault-injection test cases for the +# sqlite3_db_cacheflush API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cacheflush +source $testdir/malloc_common.tcl + +# Run the supplied SQL on a copy of the database currently stored on +# disk in file $dbfile. +proc diskquery {dbfile sql} { + forcecopy $dbfile dq.db + sqlite3 dq dq.db + set res [execsql $sql dq] + dq close + set res +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); +} +faultsim_save_and_close + +do_faultsim_test 1.1 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 1.2 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + set result [list] + db eval { SELECT * FROM t1 } { + if {$a==5} { catch { sqlite3_db_cacheflush db } } + lappend result $a $b + } + set result +} -test { + faultsim_test_result {0 {1 3 3 5 5 7 7 9}} {1 {disk I/O error}} + catch { db eval COMMIT } + faultsim_integrity_check +} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + INSERT INTO t1 VALUES(1, 2, randomblob(600)); + INSERT INTO t1 VALUES(3, 4, randomblob(600)); + INSERT INTO t1 VALUES(5, 6, randomblob(600)); + INSERT INTO t1 VALUES(7, 8, randomblob(600)); + INSERT INTO t1 VALUES(9, 10, randomblob(600)); +} +faultsim_save_and_close + +do_faultsim_test 2.1 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + set result [list] + db eval { SELECT * FROM t1 } { + if {$a==5} { catch { sqlite3_db_cacheflush db } } + lappend result $a $b + } + set result +} -test { + faultsim_test_result {0 {1 3 3 5 5 7 7 9 9 11}} {1 {disk I/O error}} + catch { db eval { INSERT INTO t1 VALUES(11, 12, randomblob(600)) } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.2 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b+1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval { SELECT * FROM t1 } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.3 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b-1; + } +} -body { + sqlite3_db_cacheflush db +} -test { + if {[sqlite3_get_autocommit db]} { error "Transaction rolled back!" } + faultsim_test_result {0 {}} {1 {disk I/O error}} + catch { db eval { INSERT INTO t1 VALUES(11, 12, randomblob(600)) } } + catch { db eval COMMIT } + faultsim_integrity_check +} + +do_faultsim_test 2.4 -prep { + faultsim_restore_and_reopen + db eval { + BEGIN; + UPDATE t1 SET b=b-1; + } +} -body { + catch { sqlite3_db_cacheflush db } + catch { sqlite3_db_release_memory db } + catch { sqlite3_db_cacheflush db } + execsql { SELECT a, b FROM t1 } +} -test { + faultsim_test_result {0 {1 1 3 3 5 5 7 7 9 9}} {1 {disk I/O error}} + catchsql ROLLBACK + faultsim_integrity_check +} + +finish_test +