Index: src/test_async.c ================================================================== --- src/test_async.c +++ src/test_async.c @@ -22,12 +22,12 @@ ** it is happening at its usual slow pace in the background. ** ** Asynchronous I/O appears to give better responsiveness, but at a price. ** You lose the Durable property. With the default I/O backend of SQLite, ** once a write completes, you know that the information you wrote is -** safely on disk. With the asynchronous I/O, this is no the case. If -** your program crashes or if you take a power lose after the database +** safely on disk. With the asynchronous I/O, this is not the case. If +** your program crashes or if a power lose occurs after the database ** write but before the asynchronous write thread has completed, then the ** database change might never make it to disk and the next user of the ** database might not see your change. ** ** You lose Durability with asynchronous I/O, but you still retain the @@ -34,30 +34,18 @@ ** other parts of ACID: Atomic, Consistent, and Isolated. Many ** appliations get along fine without the Durablity. ** ** HOW IT WORKS ** -** Asynchronous I/O works by overloading the OS-layer disk I/O routines -** with modified versions that store the data to be written in queue of -** pending write operations. Look at the asyncEnable() subroutine to see -** how overloading works. Six os-layer routines are overloaded: +** Asynchronous I/O works by creating a special SQLite "vfs" structure +** and registering it with sqlite3_vfs_register(). When files opened via +** this vfs are written to (using sqlite3OsWrite()), the data is not +** written directly to disk, but is placed in the "write-queue" to be +** handled by the background thread. ** -** sqlite3OsOpenReadWrite; -** sqlite3OsOpenReadOnly; -** sqlite3OsOpenExclusive; -** sqlite3OsDelete; -** sqlite3OsFileExists; -** sqlite3OsSyncDirectory; -** -** The original implementations of these routines are saved and are -** used by the writer thread to do the real I/O. The substitute -** implementations typically put the I/O operation on a queue -** to be handled later by the writer thread, though read operations -** must be handled right away, obviously. -** -** Asynchronous I/O is disabled by setting the os-layer interface routines -** back to their original values. +** The special vfs is registered (and unregistered) by calls to +** function asyncEnable() (see below). ** ** LIMITATIONS ** ** This demonstration code is deliberately kept simple in order to keep ** the main ideas clear and easy to understand. Real applications that @@ -71,11 +59,14 @@ */ /* ** If this symbol is defined, then file-system locks are obtained as ** required. This slows things down, but allows multiple processes -** to access the database concurrently. +** to access the database concurrently. If this symbol is not defined, +** then connections from within a single process will respect each +** others database locks, but external connections will not - leading +** to database corruption. */ #define ENABLE_FILE_LOCKING #include "sqliteInt.h" #include @@ -123,11 +114,15 @@ ** THREAD SAFETY NOTES ** ** Basic rules: ** ** * Both read and write access to the global write-op queue must be -** protected by the async.queueMutex. +** protected by the async.queueMutex. As are the async.ioError and +** async.nFile variables. +** +** * The async.aLock hash-table and all AsyncLock and AsyncFileLock +** structures must be protected by teh async.lockMutex mutex. ** ** * The file handles from the underlying system are assumed not to ** be thread safe. ** ** * See the last two paragraphs under "The Writer Thread" for @@ -306,34 +301,46 @@ int nByte; /* See above */ char *zBuf; /* Data to write to file (or NULL if op!=ASYNC_WRITE) */ AsyncWrite *pNext; /* Next write operation (to any file) */ }; +/* +** An instance of this structure is created for each distinct open file +** (i.e. if two handles are opened on the one file, only one of these +** structures is allocated) and stored in the async.aLock hash table. The +** keys for async.aLock are the full pathnames of the opened files. +** +** AsyncLock.pList points to the head of a linked list of AsyncFileLock +** structures, one for each handle currently open on the file. +** +** If the opened file is not a main-database (the SQLITE_OPEN_MAIN_DB is +** not passed to the sqlite3OsOpen() call), or if ENABLE_FILE_LOCKING is +** not defined at compile time, variables AsyncLock.pFile and +** AsyncLock.eLock are never used. Otherwise, pFile is a file handle +** opened on the file in question and used to obtain the file-system +** locks required by database connections within this process. +** +** See comments above the asyncLock() function for more details on +** the implementation of database locking used by this backend. +*/ +struct AsyncLock { + sqlite3_file *pFile; + int eLock; + AsyncFileLock *pList; +}; + /* ** An instance of the following structure is allocated along with each ** AsyncFileData structure (see AsyncFileData.lock), but is only used if the ** file was opened with the SQLITE_OPEN_MAIN_DB. -** -** The global async.aLock[] hash table maps from database file-name to a -** linked-list of AsyncFileLock structures corresponding to handles opened on -** the file. The AsyncFileLock structures are linked into the list when the -** file is opened and removed when it is closed. Mutex async.lockMutex must be -** held before accessing any AsyncFileLock structure or the async.aLock[] -** table. */ struct AsyncFileLock { int eLock; /* Internally visible lock state (sqlite pov) */ int eAsyncLock; /* Lock-state with write-queue unlock */ AsyncFileLock *pNext; }; -struct AsyncLock { - sqlite3_file *pFile; - int eLock; - AsyncFileLock *pList; -}; - /* ** The AsyncFile structure is a subclass of sqlite3_file used for ** asynchronous IO. ** ** All of the actual data for the structure is stored in the structure @@ -609,10 +616,11 @@ if( pLock->pFile ){ for(pIter=pLock->pList; pIter; pIter=pIter->pNext){ assert(pIter->eAsyncLock>=pIter->eLock); if( pIter->eAsyncLock>eRequired ){ eRequired = pIter->eAsyncLock; + assert(eRequired>=0 && eRequired<=SQLITE_LOCK_EXCLUSIVE); } } if( eRequired>pLock->eLock ){ rc = sqlite3OsLock(pLock->pFile, eRequired); }else if(eRequiredeLock){ @@ -625,16 +633,13 @@ return rc; } /* -** No disk locking is performed. We keep track of locks locally in -** the async.aLock hash table. Locking should appear to work the same -** as with standard (unmodified) SQLite as long as all connections -** come from this one process. Connections from external processes -** cannot see our internal hash table (obviously) and will thus not -** honor our locks. +** The following two methods - asyncLock() and asyncUnlock() - are used +** to obtain and release locks on database files opened with the +** asynchronous backend. */ static int asyncLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; AsyncFileData *p = ((AsyncFile *)pFile)->pData; @@ -654,13 +659,11 @@ rc = SQLITE_BUSY; } } if( rc==SQLITE_OK ){ p->lock.eLock = eLock; - if( eLock>p->lock.eAsyncLock ){ - p->lock.eAsyncLock = eLock; - } + p->lock.eAsyncLock = MAX(p->lock.eAsyncLock, eLock); } assert(p->lock.eAsyncLock>=p->lock.eLock); if( rc==SQLITE_OK ){ rc = getFileLock(pLock); } @@ -672,13 +675,11 @@ } static int asyncUnlock(sqlite3_file *pFile, int eLock){ AsyncFileData *p = ((AsyncFile *)pFile)->pData; AsyncFileLock *pLock = &p->lock; pthread_mutex_lock(&async.lockMutex); - if( pLock->eLock>eLock ){ - pLock->eLock = eLock; - } + pLock->eLock = MIN(pLock->eLock, eLock); pthread_mutex_unlock(&async.lockMutex); return addNewAsyncWrite(p, ASYNC_UNLOCK, 0, eLock, 0); } /* @@ -1008,20 +1009,23 @@ ** this routine while any SQLite database connections are open. */ static void asyncEnable(int enable){ if( enable ){ if( !async_vfs.pAppData ){ + static int hashTableInit = 0; async_vfs.pAppData = (void *)sqlite3_vfs_find(0); async_vfs.mxPathname = ((sqlite3_vfs *)async_vfs.pAppData)->mxPathname; sqlite3_vfs_register(&async_vfs, 1); - sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1); + if( !hashTableInit ){ + sqlite3HashInit(&async.aLock, SQLITE_HASH_BINARY, 1); + hashTableInit = 1; + } } }else{ if( async_vfs.pAppData ){ sqlite3_vfs_unregister(&async_vfs); async_vfs.pAppData = 0; - sqlite3HashClear(&async.aLock); } } } /* @@ -1147,18 +1151,23 @@ */ pthread_mutex_lock(&async.lockMutex); pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); for(ppIter=&pLock->pList; *ppIter; ppIter=&((*ppIter)->pNext)){ if( (*ppIter)==&pData->lock ){ - *ppIter = (*ppIter)->pNext; + *ppIter = pData->lock.pNext; break; } } if( !pLock->pList ){ - if( pLock->pFile ) sqlite3OsClose(pLock->pFile); + if( pLock->pFile ){ + sqlite3OsClose(pLock->pFile); + } sqlite3_free(pLock); sqlite3HashInsert(&async.aLock, pData->zName, pData->nName, 0); + if( !sqliteHashFirst(&async.aLock) ){ + sqlite3HashClear(&async.aLock); + } }else{ rc = getFileLock(pLock); } pthread_mutex_unlock(&async.lockMutex); @@ -1169,17 +1178,13 @@ case ASYNC_UNLOCK: { AsyncLock *pLock; AsyncFileData *pData = p->pFileData; int eLock = p->nByte; pthread_mutex_lock(&async.lockMutex); - if( pData->lock.eAsyncLock>eLock ){ - if( pData->lock.eLock>eLock ){ - pData->lock.eAsyncLock = pData->lock.eLock; - }else{ - pData->lock.eAsyncLock = eLock; - } - } + pData->lock.eAsyncLock = MIN( + pData->lock.eAsyncLock, MAX(pData->lock.eLock, eLock) + ); assert(pData->lock.eAsyncLock>=pData->lock.eLock); pLock = sqlite3HashFind(&async.aLock, pData->zName, pData->nName); rc = getFileLock(pLock); pthread_mutex_unlock(&async.lockMutex); break; Index: test/async.test ================================================================== --- test/async.test +++ test/async.test @@ -4,11 +4,11 @@ # May you share freely, never taking more than you give. # #*********************************************************************** # This file runs all tests. # -# $Id: async.test,v 1.9 2007/09/04 18:28:44 danielk1977 Exp $ +# $Id: async.test,v 1.10 2007/09/05 11:34:54 danielk1977 Exp $ if {[catch {sqlite3async_enable}]} { # The async logic is not built into this system return @@ -16,11 +16,15 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl rename finish_test really_finish_test -proc finish_test {} {} +proc finish_test {} { + catch {db close} + catch {db2 close} + catch {db3 close} +} set ISQUICK 1 set INCLUDE { select1.test select2.test @@ -32,11 +36,10 @@ trans.test lock.test lock3.test lock2.test } -# set INCLUDE lock4.test # Enable asynchronous IO. sqlite3async_enable 1 rename do_test really_do_test @@ -49,11 +52,19 @@ foreach testfile [lsort -dictionary [glob $testdir/*.test]] { set tail [file tail $testfile] if {[lsearch -exact $INCLUDE $tail]<0} continue source $testfile - catch {db close} + + # Make sure everything is flushed through. This is because [source]ing + # the next test file will delete the database file on disk (using + # [file delete]). If the asynchronous backend still has the file + # open, it will become confused. + # + sqlite3async_halt idle + sqlite3async_start + sqlite3async_wait } # Flush the write-queue and disable asynchronous IO. This should ensure # all allocated memory is cleaned up. set sqlite3async_trace 1 Index: test/async2.test ================================================================== --- test/async2.test +++ test/async2.test @@ -3,11 +3,11 @@ # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # -# $Id: async2.test,v 1.6 2007/08/30 10:49:55 danielk1977 Exp $ +# $Id: async2.test,v 1.7 2007/09/05 11:34:54 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -68,11 +68,12 @@ malloc-transient { sqlite3_memdebug_fail $n -repeat 0 } } sqlite3async_halt idle sqlite3async_start sqlite3async_wait - + sqlite3async_enable 0 + set ::sqlite_io_error_pending 0 sqlite3_memdebug_fail -1 sqlite3 db test.db set c [db eval {SELECT c FROM counter LIMIT 1}] @@ -111,15 +112,15 @@ FIN { set ::go 0 } } - sqlite3async_enable 0 + db close } } catch {db close} sqlite3async_halt idle sqlite3async_start sqlite3async_wait finish_test