Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -413,10 +413,11 @@ $(TOP)/src/test_window.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/session/sqlite3changebatch.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ $(TOP)/ext/intck/test_intck.c \ $(TOP)/ext/intck/sqlite3intck.c \ @@ -512,12 +513,13 @@ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c \ - $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/misc/stmt.c \ + $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/test_session.c \ fts5.c # Header files used by all library source files. # HDR = \ Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1560,11 +1560,12 @@ $(TOP)\src\test_window.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c \ $(TOP)\ext\rbu\test_rbu.c \ - $(TOP)\ext\session\test_session.c + $(TOP)\ext\session\test_session.c \ + $(TOP)\ext\session\sqlite3changebatch.c # Statically linked extensions. # TESTEXT = \ $(TOP)\ext\expert\sqlite3expert.c \ ADDED doc/begin_concurrent.md Index: doc/begin_concurrent.md ================================================================== --- /dev/null +++ doc/begin_concurrent.md @@ -0,0 +1,106 @@ + +Begin Concurrent +================ + +## Overview + +Usually, SQLite allows at most one writer to proceed concurrently. The +BEGIN CONCURRENT enhancement allows multiple writers to process write +transactions simultanously if the database is in "wal" or "wal2" mode, +although the system still serializes COMMIT commands. + +When a write-transaction is opened with "BEGIN CONCURRENT", actually +locking the database is deferred until a COMMIT is executed. This means +that any number of transactions started with BEGIN CONCURRENT may proceed +concurrently. The system uses optimistic page-level-locking to prevent +conflicting concurrent transactions from being committed. + +When a BEGIN CONCURRENT transaction is committed, the system checks whether +or not any of the database pages that the transaction has read have been +modified since the BEGIN CONCURRENT was opened. In other words - it asks +if the transaction being committed operates on a different set of data than +all other concurrently executing transactions. If the answer is "yes, this +transaction did not read or modify any data modified by any concurrent +transaction", then the transaction is committed as normal. Otherwise, if the +transaction does conflict, it cannot be committed and an SQLITE_BUSY_SNAPSHOT +error is returned. At this point, all the client can do is ROLLBACK the +transaction. + +If SQLITE_BUSY_SNAPSHOT is returned, messages are output via the sqlite3_log +mechanism indicating the page and table or index on which the conflict +occurred. This can be useful when optimizing concurrency. + +## Application Programming Notes + +In order to serialize COMMIT processing, SQLite takes a lock on the database +as part of each COMMIT command and releases it before returning. At most one +writer may hold this lock at any one time. If a writer cannot obtain the lock, +it uses SQLite's busy-handler to pause and retry for a while: + + + https://www.sqlite.org/c3ref/busy_handler.html + + +If there is significant contention for the writer lock, this mechanism can be +inefficient. In this case it is better for the application to use a mutex or +some other mechanism that supports blocking to ensure that at most one writer +is attempting to COMMIT a BEGIN CONCURRENT transaction at a time. This is +usually easier if all writers are part of the same operating system process. + +If all database clients (readers and writers) are located in the same OS +process, and if that OS is a Unix variant, then it can be more efficient to +the built-in VFS "unix-excl" instead of the default "unix". This is because it +uses more efficient locking primitives. + +The key to maximizing concurrency using BEGIN CONCURRENT is to ensure that +there are a large number of non-conflicting transactions. In SQLite, each +table and each index is stored as a separate b-tree, each of which is +distributed over a discrete set of database pages. This means that: + + * Two transactions that write to different sets of tables never + conflict, and that + + * Two transactions that write to the same tables or indexes only + conflict if the values of the keys (either primary keys or indexed + rows) are fairly close together. For example, given a large + table with the schema: + +
     CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);
+ + writing two rows with adjacent values for "a" probably will cause a + conflict (as the two keys are stored on the same page), but writing two + rows with vastly different values for "a" will not (as the keys will likly + be stored on different pages). + +Note that, in SQLite, if values are not explicitly supplied for an INTEGER +PRIMARY KEY, as for example in: + +> + INSERT INTO t1(b) VALUES(); + +then monotonically increasing values are assigned automatically. This is +terrible for concurrency, as it all but ensures that all new rows are +added to the same database page. In such situations, it is better to +explicitly assign random values to INTEGER PRIMARY KEY fields. + +This problem also comes up for non-WITHOUT ROWID tables that do not have an +explicit INTEGER PRIMARY KEY column. In these cases each table has an implicit +INTEGER PRIMARY KEY column that is assigned increasing values, leading to the +same problem as omitting to assign a value to an explicit INTEGER PRIMARY KEY +column. + +For both explicit and implicit INTEGER PRIMARY KEYs, it is possible to have +SQLite assign values at random (instead of the monotonically increasing +values) by writing a row with a rowid equal to the largest possible signed +64-bit integer to the table. For example: + + INSERT INTO t1(a) VALUES(9223372036854775807); + +Applications should take care not to malfunction due to the presence of such +rows. + +The nature of some types of indexes, for example indexes on timestamp fields, +can also cause problems (as concurrent transactions may assign similar +timestamps that will be stored on the same db page to new records). In these +cases the database schema may need to be rethought to increase the concurrency +provided by page-level-locking. ADDED ext/session/changebatch1.test Index: ext/session/changebatch1.test ================================================================== --- /dev/null +++ ext/session/changebatch1.test @@ -0,0 +1,222 @@ +# 2016 August 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !session {finish_test; return} + +set testprefix changebatch1 + + +proc sql_to_changeset {method sql} { + sqlite3session S db main + S attach * + execsql $sql + set ret [S $method] + S delete + return $ret +} + +proc do_changebatch_test {tn method args} { + set C [list] + foreach a $args { + lappend C [sql_to_changeset $method $a] + } + + sqlite3changebatch cb db + set i 1 + foreach ::cs [lrange $C 0 end-1] { + set rc [cb add $::cs] + if {$rc!="SQLITE_OK"} { error "expected SQLITE_OK, got $rc (i=$i)" } + incr i + } + + set ::cs [lindex $C end] + do_test $tn { cb add [set ::cs] } SQLITE_CONSTRAINT + cb delete +} + +proc do_changebatch_test1 {tn args} { + uplevel do_changebatch_test $tn changeset $args +} +proc do_changebatch_test2 {tn args} { + uplevel do_changebatch_test $tn fullchangeset $args +} + +#------------------------------------------------------------------------- +# The body of the following loop contains tests for database schemas +# that do not feature multi-column UNIQUE constraints. In this case +# it doesn't matter if the changesets are generated using +# sqlite3session_changeset() or sqlite3session_fullchangeset(). +# +foreach {tn testfunction} { + 1 do_changebatch_test1 + 2 do_changebatch_test2 +} { + reset_db + + #------------------------------------------------------------------------- + # + do_execsql_test $tn.1.0 { + CREATE TABLE t1(a PRIMARY KEY, b); + } + + $testfunction $tn.1.1 { + INSERT INTO t1 VALUES(1, 1); + } { + DELETE FROM t1 WHERE a=1; + } + + do_execsql_test $tn.1.2.0 { + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + } + $testfunction $tn.1.2.1 { + DELETE FROM t1 WHERE a=2; + } { + INSERT INTO t1 VALUES(2, 2); + } + + #------------------------------------------------------------------------- + # + do_execsql_test $tn.2.0 { + CREATE TABLE x1(a, b PRIMARY KEY, c UNIQUE); + CREATE TABLE x2(a PRIMARY KEY, b UNIQUE, c UNIQUE); + CREATE INDEX x1a ON x1(a); + + INSERT INTO x1 VALUES(1, 1, 'a'); + INSERT INTO x1 VALUES(1, 2, 'b'); + INSERT INTO x1 VALUES(1, 3, 'c'); + } + + $testfunction $tn.2.1 { + DELETE FROM x1 WHERE b=2; + } { + UPDATE x1 SET c='b' WHERE b=3; + } + + $testfunction $tn.2.2 { + DELETE FROM x1 WHERE b=1; + } { + INSERT INTO x1 VALUES(1, 5, 'a'); + } + + set L [list] + for {set i 1000} {$i < 10000} {incr i} { + lappend L "INSERT INTO x2 VALUES($i, $i, 'x' || $i)" + } + lappend L "DELETE FROM x2 WHERE b=1005" + $testfunction $tn.2.3 {*}$L + + execsql { INSERT INTO x1 VALUES('f', 'f', 'f') } + $testfunction $tn.2.4 { + INSERT INTO x2 VALUES('f', 'f', 'f'); + } { + INSERT INTO x1 VALUES('g', 'g', 'g'); + } { + DELETE FROM x1 WHERE b='f'; + } { + INSERT INTO x2 VALUES('g', 'g', 'g'); + } { + INSERT INTO x1 VALUES('f', 'f', 'f'); + } + + execsql { + DELETE FROM x1; + INSERT INTO x1 VALUES(1.5, 1.5, 1.5); + } + $testfunction $tn.2.5 { + DELETE FROM x1 WHERE b BETWEEN 1 AND 2; + } { + INSERT INTO x1 VALUES(2.5, 2.5, 2.5); + } { + INSERT INTO x1 VALUES(1.5, 1.5, 1.5); + } + + execsql { + DELETE FROM x2; + INSERT INTO x2 VALUES(X'abcd', X'1234', X'7890'); + INSERT INTO x2 VALUES(X'0000', X'0000', X'0000'); + } + breakpoint + $testfunction $tn.2.6 { + UPDATE x2 SET c = X'1234' WHERE a=X'abcd'; + INSERT INTO x2 VALUES(X'1234', X'abcd', X'7890'); + } { + DELETE FROM x2 WHERE b=X'0000'; + } { + INSERT INTO x2 VALUES(1, X'0000', NULL); + } +} + +#------------------------------------------------------------------------- +# Test some multi-column UNIQUE constraints. First Using _changeset() to +# demonstrate the problem, then using _fullchangeset() to show that it has +# been fixed. +# +reset_db +do_execsql_test 3.0 { + CREATE TABLE y1(a PRIMARY KEY, b, c, UNIQUE(b, c)); + INSERT INTO y1 VALUES(1, 1, 1); + INSERT INTO y1 VALUES(2, 2, 2); + INSERT INTO y1 VALUES(3, 3, 3); + INSERT INTO y1 VALUES(4, 3, 4); + BEGIN; +} + +do_test 3.1.1 { + set c1 [sql_to_changeset changeset { DELETE FROM y1 WHERE a=4 }] + set c2 [sql_to_changeset changeset { UPDATE y1 SET c=4 WHERE a=3 }] + sqlite3changebatch cb db + cb add $c1 + cb add $c2 +} {SQLITE_OK} +do_test 3.1.2 { + cb delete + execsql ROLLBACK +} {} + +do_test 3.1.1 { + set c1 [sql_to_changeset fullchangeset { DELETE FROM y1 WHERE a=4 }] + set c2 [sql_to_changeset fullchangeset { UPDATE y1 SET c=4 WHERE a=3 }] + sqlite3changebatch cb db + cb add $c1 + cb add $c2 +} {SQLITE_OK} +do_test 3.1.2 { + cb delete +} {} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y, z, PRIMARY KEY(x, y), UNIQUE(z)); +} + +do_test 4.1 { + set c1 [sql_to_changeset fullchangeset { INSERT INTO t1 VALUES(1, 2, 3) }] + execsql { + DROP TABLE t1; + CREATE TABLE t1(w, x, y, z, PRIMARY KEY(x, y), UNIQUE(z)); + } + sqlite3changebatch cb db + list [catch { cb add $c1 } msg] $msg +} {1 SQLITE_RANGE} + +cb delete + +finish_test ADDED ext/session/changebatchfault.test Index: ext/session/changebatchfault.test ================================================================== --- /dev/null +++ ext/session/changebatchfault.test @@ -0,0 +1,42 @@ +# 2011 Mar 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# The focus of this file is testing the session module. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] session_common.tcl] +source $testdir/tester.tcl +ifcapable !session {finish_test; return} +set testprefix changebatchfault + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c PRIMARY KEY, UNIQUE(a, b)); + INSERT INTO t1 VALUES('a', 'a', 'a'); + INSERT INTO t1 VALUES('b', 'b', 'b'); +} + +set ::c1 [changeset_from_sql { delete from t1 where c='a' }] +set ::c2 [changeset_from_sql { insert into t1 values('c', 'c', 'c') }] + +do_faultsim_test 1 -faults oom-* -body { + sqlite3changebatch cb db + cb add $::c1 + cb add $::c2 +} -test { + faultsim_test_result {0 SQLITE_OK} {1 SQLITE_NOMEM} + catch { cb delete } +} + + +finish_test Index: ext/session/sessionH.test ================================================================== --- ext/session/sessionH.test +++ ext/session/sessionH.test @@ -27,11 +27,11 @@ } do_then_apply_sql -ignorenoop { WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERe i<10000 ) - INSERT INTO t1 SELECT 'abcde', randomblob(16), i FROM s; + INSERT INTO t1 SELECT 'abcde', randomblob(18), i FROM s; } compare_db db db2 } {} #------------------------------------------------------------------------ ADDED ext/session/sqlite3changebatch.c Index: ext/session/sqlite3changebatch.c ================================================================== --- /dev/null +++ ext/session/sqlite3changebatch.c @@ -0,0 +1,485 @@ + +#if !defined(SQLITE_TEST) || (defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK)) + +#include "sqlite3session.h" +#include "sqlite3changebatch.h" + +#include +#include + +typedef struct BatchTable BatchTable; +typedef struct BatchIndex BatchIndex; +typedef struct BatchIndexEntry BatchIndexEntry; +typedef struct BatchHash BatchHash; + +struct sqlite3_changebatch { + sqlite3 *db; /* Database handle used to read schema */ + BatchTable *pTab; /* First in linked list of tables */ + int iChangesetId; /* Current changeset id */ + int iNextIdxId; /* Next available index id */ + int nEntry; /* Number of entries in hash table */ + int nHash; /* Number of hash buckets */ + BatchIndexEntry **apHash; /* Array of hash buckets */ +}; + +struct BatchTable { + BatchIndex *pIdx; /* First in linked list of UNIQUE indexes */ + BatchTable *pNext; /* Next table */ + char zTab[1]; /* Table name */ +}; + +struct BatchIndex { + BatchIndex *pNext; /* Next index on same table */ + int iId; /* Index id (assigned internally) */ + int bPk; /* True for PK index */ + int nCol; /* Size of aiCol[] array */ + int *aiCol; /* Array of columns that make up index */ +}; + +struct BatchIndexEntry { + BatchIndexEntry *pNext; /* Next colliding hash table entry */ + int iChangesetId; /* Id of associated changeset */ + int iIdxId; /* Id of index this key is from */ + int szRecord; + char aRecord[1]; +}; + +/* +** Allocate and zero a block of nByte bytes. Must be freed using cbFree(). +*/ +static void *cbMalloc(int *pRc, int nByte){ + void *pRet; + + if( *pRc ){ + pRet = 0; + }else{ + pRet = sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + *pRc = SQLITE_NOMEM; + } + } + + return pRet; +} + +/* +** Free an allocation made by cbMalloc(). +*/ +static void cbFree(void *p){ + sqlite3_free(p); +} + +/* +** Return the hash bucket that pEntry belongs in. +*/ +static int cbHash(sqlite3_changebatch *p, BatchIndexEntry *pEntry){ + unsigned int iHash = (unsigned int)pEntry->iIdxId; + unsigned char *pEnd = (unsigned char*)&pEntry->aRecord[pEntry->szRecord]; + unsigned char *pIter; + + for(pIter=(unsigned char*)pEntry->aRecord; pIternHash); +} + +/* +** Resize the hash table. +*/ +static int cbHashResize(sqlite3_changebatch *p){ + int rc = SQLITE_OK; + BatchIndexEntry **apNew; + int nNew = (p->nHash ? p->nHash*2 : 512); + int i; + + apNew = cbMalloc(&rc, sizeof(BatchIndexEntry*) * nNew); + if( rc==SQLITE_OK ){ + int nHash = p->nHash; + p->nHash = nNew; + for(i=0; iapHash[i])!=0 ){ + int iHash = cbHash(p, pEntry); + p->apHash[i] = pEntry->pNext; + pEntry->pNext = apNew[iHash]; + apNew[iHash] = pEntry; + } + } + + cbFree(p->apHash); + p->apHash = apNew; + } + + return rc; +} + + +/* +** Allocate a new sqlite3_changebatch object. +*/ +int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp){ + sqlite3_changebatch *pRet; + int rc = SQLITE_OK; + *pp = pRet = (sqlite3_changebatch*)cbMalloc(&rc, sizeof(sqlite3_changebatch)); + if( pRet ){ + pRet->db = db; + } + return rc; +} + +/* +** Add a BatchIndex entry for index zIdx to table pTab. +*/ +static int cbAddIndex( + sqlite3_changebatch *p, + BatchTable *pTab, + const char *zIdx, + int bPk +){ + int nCol = 0; + sqlite3_stmt *pIndexInfo = 0; + BatchIndex *pNew = 0; + int rc; + char *zIndexInfo; + + zIndexInfo = (char*)sqlite3_mprintf("PRAGMA main.index_info = %Q", zIdx); + if( zIndexInfo ){ + rc = sqlite3_prepare_v2(p->db, zIndexInfo, -1, &pIndexInfo, 0); + sqlite3_free(zIndexInfo); + }else{ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ nCol++; } + rc = sqlite3_reset(pIndexInfo); + } + + pNew = (BatchIndex*)cbMalloc(&rc, sizeof(BatchIndex) + sizeof(int) * nCol); + if( rc==SQLITE_OK ){ + pNew->nCol = nCol; + pNew->bPk = bPk; + pNew->aiCol = (int*)&pNew[1]; + pNew->iId = p->iNextIdxId++; + while( SQLITE_ROW==sqlite3_step(pIndexInfo) ){ + int i = sqlite3_column_int(pIndexInfo, 0); + int j = sqlite3_column_int(pIndexInfo, 1); + pNew->aiCol[i] = j; + } + rc = sqlite3_reset(pIndexInfo); + } + + if( rc==SQLITE_OK ){ + pNew->pNext = pTab->pIdx; + pTab->pIdx = pNew; + }else{ + cbFree(pNew); + } + sqlite3_finalize(pIndexInfo); + + return rc; +} + +/* +** Free the object passed as the first argument. +*/ +static void cbFreeTable(BatchTable *pTab){ + BatchIndex *pIdx; + BatchIndex *pIdxNext; + for(pIdx=pTab->pIdx; pIdx; pIdx=pIdxNext){ + pIdxNext = pIdx->pNext; + cbFree(pIdx); + } + cbFree(pTab); +} + +/* +** Find or create the BatchTable object named zTab. +*/ +static int cbFindTable( + sqlite3_changebatch *p, + const char *zTab, + BatchTable **ppTab +){ + BatchTable *pRet = 0; + int rc = SQLITE_OK; + + for(pRet=p->pTab; pRet; pRet=pRet->pNext){ + if( 0==sqlite3_stricmp(zTab, pRet->zTab) ) break; + } + + if( pRet==0 ){ + int nTab = strlen(zTab); + pRet = (BatchTable*)cbMalloc(&rc, nTab + sizeof(BatchTable)); + if( pRet ){ + sqlite3_stmt *pIndexList = 0; + char *zIndexList = 0; + int rc2; + memcpy(pRet->zTab, zTab, nTab); + + zIndexList = sqlite3_mprintf("PRAGMA main.index_list = %Q", zTab); + if( zIndexList==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zIndexList, -1, &pIndexList, 0); + sqlite3_free(zIndexList); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pIndexList) ){ + if( sqlite3_column_int(pIndexList, 2) ){ + const char *zIdx = (const char*)sqlite3_column_text(pIndexList, 1); + const char *zTyp = (const char*)sqlite3_column_text(pIndexList, 3); + rc = cbAddIndex(p, pRet, zIdx, (zTyp[0]=='p')); + } + } + rc2 = sqlite3_finalize(pIndexList); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc==SQLITE_OK ){ + pRet->pNext = p->pTab; + p->pTab = pRet; + }else{ + cbFreeTable(pRet); + pRet = 0; + } + } + } + + *ppTab = pRet; + return rc; +} + +/* +** Extract value iVal from the changeset iterator passed as the first +** argument. Set *ppVal to point to the value before returning. +** +** This function attempts to extract the value using function xVal +** (which is always either sqlite3changeset_new or sqlite3changeset_old). +** If the call returns SQLITE_OK but does not supply an sqlite3_value* +** pointer, an attempt to extract the value is made using the xFallback +** function. +*/ +static int cbGetChangesetValue( + sqlite3_changeset_iter *pIter, + int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), + int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**), + int iVal, + sqlite3_value **ppVal +){ + int rc = xVal(pIter, iVal, ppVal); + if( rc==SQLITE_OK && *ppVal==0 && xFallback ){ + rc = xFallback(pIter, iVal, ppVal); + } + return rc; +} + +static int cbAddToHash( + sqlite3_changebatch *p, + sqlite3_changeset_iter *pIter, + BatchIndex *pIdx, + int (*xVal)(sqlite3_changeset_iter*,int,sqlite3_value**), + int (*xFallback)(sqlite3_changeset_iter*,int,sqlite3_value**), + int *pbConf +){ + BatchIndexEntry *pNew; + int sz = pIdx->nCol; + int i; + int iOut = 0; + int rc = SQLITE_OK; + + for(i=0; rc==SQLITE_OK && inCol; i++){ + sqlite3_value *pVal; + rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal); + if( rc==SQLITE_OK ){ + int eType = 0; + if( pVal ) eType = sqlite3_value_type(pVal); + switch( eType ){ + case 0: + case SQLITE_NULL: + return SQLITE_OK; + + case SQLITE_INTEGER: + sz += 8; + break; + case SQLITE_FLOAT: + sz += 8; + break; + + default: + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + sz += sqlite3_value_bytes(pVal); + break; + } + } + } + + pNew = cbMalloc(&rc, sizeof(BatchIndexEntry) + sz); + if( pNew ){ + pNew->iChangesetId = p->iChangesetId; + pNew->iIdxId = pIdx->iId; + pNew->szRecord = sz; + + for(i=0; inCol; i++){ + int eType; + sqlite3_value *pVal; + rc = cbGetChangesetValue(pIter, xVal, xFallback, pIdx->aiCol[i], &pVal); + if( rc!=SQLITE_OK ) break; /* coverage: condition is never true */ + eType = sqlite3_value_type(pVal); + pNew->aRecord[iOut++] = eType; + switch( eType ){ + case SQLITE_INTEGER: { + sqlite3_int64 i64 = sqlite3_value_int64(pVal); + memcpy(&pNew->aRecord[iOut], &i64, 8); + iOut += 8; + break; + } + case SQLITE_FLOAT: { + double d64 = sqlite3_value_double(pVal); + memcpy(&pNew->aRecord[iOut], &d64, sizeof(double)); + iOut += sizeof(double); + break; + } + + default: { + int nByte = sqlite3_value_bytes(pVal); + const char *z = (const char*)sqlite3_value_blob(pVal); + memcpy(&pNew->aRecord[iOut], z, nByte); + iOut += nByte; + break; + } + } + } + } + + if( rc==SQLITE_OK && p->nEntry>=(p->nHash/2) ){ + rc = cbHashResize(p); + } + + if( rc==SQLITE_OK ){ + BatchIndexEntry *pIter; + int iHash = cbHash(p, pNew); + + assert( iHash>=0 && iHashnHash ); + for(pIter=p->apHash[iHash]; pIter; pIter=pIter->pNext){ + if( pNew->szRecord==pIter->szRecord + && 0==memcmp(pNew->aRecord, pIter->aRecord, pNew->szRecord) + ){ + if( pNew->iChangesetId!=pIter->iChangesetId ){ + *pbConf = 1; + } + cbFree(pNew); + pNew = 0; + break; + } + } + + if( pNew ){ + pNew->pNext = p->apHash[iHash]; + p->apHash[iHash] = pNew; + p->nEntry++; + } + }else{ + cbFree(pNew); + } + + return rc; +} + + +/* +** Add a changeset to the current batch. +*/ +int sqlite3changebatch_add(sqlite3_changebatch *p, void *pBuf, int nBuf){ + sqlite3_changeset_iter *pIter; /* Iterator opened on pBuf/nBuf */ + int rc; /* Return code */ + int bConf = 0; /* Conflict was detected */ + + rc = sqlite3changeset_start(&pIter, nBuf, pBuf); + if( rc==SQLITE_OK ){ + int rc2; + for(rc2 = sqlite3changeset_next(pIter); + rc2==SQLITE_ROW; + rc2 = sqlite3changeset_next(pIter) + ){ + BatchTable *pTab; + BatchIndex *pIdx; + const char *zTab; /* Table this change applies to */ + int nCol; /* Number of columns in table */ + int op; /* UPDATE, INSERT or DELETE */ + + sqlite3changeset_op(pIter, &zTab, &nCol, &op, 0); + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); + + rc = cbFindTable(p, zTab, &pTab); + assert( pTab || rc!=SQLITE_OK ); + if( pTab ){ + for(pIdx=pTab->pIdx; pIdx && rc==SQLITE_OK; pIdx=pIdx->pNext){ + if( op==SQLITE_UPDATE && pIdx->bPk ) continue; + if( op==SQLITE_UPDATE || op==SQLITE_DELETE ){ + rc = cbAddToHash(p, pIter, pIdx, sqlite3changeset_old, 0, &bConf); + } + if( op==SQLITE_UPDATE || op==SQLITE_INSERT ){ + rc = cbAddToHash(p, pIter, pIdx, + sqlite3changeset_new, sqlite3changeset_old, &bConf + ); + } + } + } + if( rc!=SQLITE_OK ) break; + } + + rc2 = sqlite3changeset_finalize(pIter); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK && bConf ){ + rc = SQLITE_CONSTRAINT; + } + p->iChangesetId++; + return rc; +} + +/* +** Zero an existing changebatch object. +*/ +void sqlite3changebatch_zero(sqlite3_changebatch *p){ + int i; + for(i=0; inHash; i++){ + BatchIndexEntry *pEntry; + BatchIndexEntry *pNext; + for(pEntry=p->apHash[i]; pEntry; pEntry=pNext){ + pNext = pEntry->pNext; + cbFree(pEntry); + } + } + cbFree(p->apHash); + p->nHash = 0; + p->apHash = 0; +} + +/* +** Delete a changebatch object. +*/ +void sqlite3changebatch_delete(sqlite3_changebatch *p){ + BatchTable *pTab; + BatchTable *pTabNext; + + sqlite3changebatch_zero(p); + for(pTab=p->pTab; pTab; pTab=pTabNext){ + pTabNext = pTab->pNext; + cbFreeTable(pTab); + } + cbFree(p); +} + +/* +** Return the db handle. +*/ +sqlite3 *sqlite3changebatch_db(sqlite3_changebatch *p){ + return p->db; +} + +#endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ ADDED ext/session/sqlite3changebatch.h Index: ext/session/sqlite3changebatch.h ================================================================== --- /dev/null +++ ext/session/sqlite3changebatch.h @@ -0,0 +1,82 @@ + +#if !defined(SQLITECHANGEBATCH_H_) +#define SQLITECHANGEBATCH_H_ 1 + +typedef struct sqlite3_changebatch sqlite3_changebatch; + +/* +** Create a new changebatch object for detecting conflicts between +** changesets associated with a schema equivalent to that of the "main" +** database of the open database handle db passed as the first +** parameter. It is the responsibility of the caller to ensure that +** the database handle is not closed until after the changebatch +** object has been deleted. +** +** A changebatch object is used to detect batches of non-conflicting +** changesets. Changesets that do not conflict may be applied to the +** target database in any order without affecting the final state of +** the database. +** +** The changebatch object only works reliably if PRIMARY KEY and UNIQUE +** constraints on tables affected by the changesets use collation +** sequences that are equivalent to built-in collation sequence +** BINARY for the == operation. +** +** If successful, SQLITE_OK is returned and (*pp) set to point to +** the new changebatch object. If an error occurs, an SQLite error +** code is returned and the final value of (*pp) is undefined. +*/ +int sqlite3changebatch_new(sqlite3 *db, sqlite3_changebatch **pp); + +/* +** Argument p points to a buffer containing a changeset n bytes in +** size. Assuming no error occurs, this function returns SQLITE_OK +** if the changeset does not conflict with any changeset passed +** to an sqlite3changebatch_add() call made on the same +** sqlite3_changebatch* handle since the most recent call to +** sqlite3changebatch_zero(). If the changeset does conflict with +** an earlier such changeset, SQLITE_CONSTRAINT is returned. Or, +** if an error occurs, some other SQLite error code may be returned. +** +** One changeset is said to conflict with another if +** either: +** +** * the two changesets contain operations (INSERT, UPDATE or +** DELETE) on the same row, identified by primary key, or +** +** * the two changesets contain operations (INSERT, UPDATE or +** DELETE) on rows with identical values in any combination +** of fields constrained by a UNIQUE constraint. +** +** Even if this function returns SQLITE_CONFLICT, the current +** changeset is added to the internal data structures - so future +** calls to this function may conflict with it. If this function +** returns any result code other than SQLITE_OK or SQLITE_CONFLICT, +** the result of any future call to sqlite3changebatch_add() is +** undefined. +** +** Only changesets may be passed to this function. Passing a +** patchset to this function results in an SQLITE_MISUSE error. +*/ +int sqlite3changebatch_add(sqlite3_changebatch*, void *p, int n); + +/* +** Zero a changebatch object. This causes the records of all earlier +** calls to sqlite3changebatch_add() to be discarded. +*/ +void sqlite3changebatch_zero(sqlite3_changebatch*); + +/* +** Return a copy of the first argument passed to the sqlite3changebatch_new() +** call used to create the changebatch object passed as the only argument +** to this function. +*/ +sqlite3 *sqlite3changebatch_db(sqlite3_changebatch*); + +/* +** Delete a changebatch object. +*/ +void sqlite3changebatch_delete(sqlite3_changebatch*); + +#endif /* !defined(SQLITECHANGEBATCH_H_) */ + Index: ext/session/sqlite3session.c ================================================================== --- ext/session/sqlite3session.c +++ ext/session/sqlite3session.c @@ -25,10 +25,17 @@ # endif #endif #define SESSIONS_ROWID "_rowid_" +/* +** The three different types of changesets generated. +*/ +#define SESSIONS_PATCHSET 0 +#define SESSIONS_CHANGESET 1 +#define SESSIONS_FULLCHANGESET 2 + static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; typedef struct SessionHook SessionHook; struct SessionHook { void *pCtx; @@ -2602,11 +2609,11 @@ ** original values of any fields that have been modified. The new.* record ** contains the new values of only those fields that have been modified. */ static int sessionAppendUpdate( SessionBuffer *pBuf, /* Buffer to append to */ - int bPatchset, /* True for "patchset", 0 for "changeset" */ + int ePatchset, /* True for "patchset", 0 for "changeset" */ sqlite3_stmt *pStmt, /* Statement handle pointing at new row */ SessionChange *p, /* Object containing old values */ u8 *abPK /* Boolean array - true for PK columns */ ){ int rc = SQLITE_OK; @@ -2666,21 +2673,21 @@ /* If at least one field has been modified, this is not a no-op. */ if( bChanged ) bNoop = 0; /* Add a field to the old.* record. This is omitted if this module is ** currently generating a patchset. */ - if( bPatchset==0 ){ - if( bChanged || abPK[i] ){ + if( ePatchset!=SESSIONS_PATCHSET ){ + if( ePatchset==SESSIONS_FULLCHANGESET || bChanged || abPK[i] ){ sessionAppendBlob(pBuf, pCsr, nAdvance, &rc); }else{ sessionAppendByte(pBuf, 0, &rc); } } /* Add a field to the new.* record. Or the only record if currently ** generating a patchset. */ - if( bChanged || (bPatchset && abPK[i]) ){ + if( bChanged || (ePatchset==SESSIONS_PATCHSET && abPK[i]) ){ sessionAppendCol(&buf2, pStmt, i, &rc); }else{ sessionAppendByte(&buf2, 0, &rc); } @@ -2702,21 +2709,21 @@ ** the changeset format if argument bPatchset is zero, or the patchset ** format otherwise. */ static int sessionAppendDelete( SessionBuffer *pBuf, /* Buffer to append to */ - int bPatchset, /* True for "patchset", 0 for "changeset" */ + int eChangeset, /* One of SESSIONS_CHANGESET etc. */ SessionChange *p, /* Object containing old values */ int nCol, /* Number of columns in table */ u8 *abPK /* Boolean array - true for PK columns */ ){ int rc = SQLITE_OK; sessionAppendByte(pBuf, SQLITE_DELETE, &rc); sessionAppendByte(pBuf, p->bIndirect, &rc); - if( bPatchset==0 ){ + if( eChangeset!=SESSIONS_PATCHSET ){ sessionAppendBlob(pBuf, p->aRecord, p->nRecord, &rc); }else{ int i; u8 *a = p->aRecord; for(i=0; inCol, pRc); sessionAppendBlob(pBuf, pTab->abPK, pTab->nCol, pRc); sessionAppendBlob(pBuf, (u8 *)pTab->zName, (int)strlen(pTab->zName)+1, pRc); } @@ -2961,11 +2968,11 @@ ** occurs, an SQLite error code is returned and both output variables set ** to 0. */ static int sessionGenerateChangeset( sqlite3_session *pSession, /* Session object */ - int bPatchset, /* True for patchset, false for changeset */ + int ePatchset, /* One of SESSIONS_CHANGESET etc. */ int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut, /* First argument for xOutput */ int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ void **ppChangeset /* OUT: Buffer containing changeset */ ){ @@ -3006,11 +3013,11 @@ if( rc==SQLITE_OK && pTab->nCol!=nOldCol ){ rc = sessionUpdateChanges(pSession, pTab); } /* Write a table header */ - sessionAppendTableHdr(&buf, bPatchset, pTab, &rc); + sessionAppendTableHdr(&buf, ePatchset, pTab, &rc); /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt(db, 0, pSession->zDb, zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel @@ -3032,14 +3039,14 @@ for(iCol=0; iColnCol; iCol++){ sessionAppendCol(&buf, pSel, iCol, &rc); } }else{ assert( pTab->abPK!=0 ); - rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, pTab->abPK); + rc = sessionAppendUpdate(&buf, ePatchset, pSel, p, pTab->abPK); } }else if( p->op!=SQLITE_INSERT ){ - rc = sessionAppendDelete(&buf, bPatchset, p, pTab->nCol,pTab->abPK); + rc = sessionAppendDelete(&buf, ePatchset, p, pTab->nCol,pTab->abPK); } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); } @@ -3094,11 +3101,12 @@ void **ppChangeset /* OUT: Buffer containing changeset */ ){ int rc; if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; - rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); + rc = sessionGenerateChangeset( + pSession, SESSIONS_CHANGESET, 0, 0, pnChangeset, ppChangeset); assert( rc || pnChangeset==0 || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize ); return rc; } @@ -3110,11 +3118,12 @@ sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ if( xOutput==0 ) return SQLITE_MISUSE; - return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0); + return sessionGenerateChangeset( + pSession, SESSIONS_CHANGESET, xOutput, pOut, 0, 0); } /* ** Streaming version of sqlite3session_patchset(). */ @@ -3122,11 +3131,12 @@ sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ if( xOutput==0 ) return SQLITE_MISUSE; - return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0); + return sessionGenerateChangeset( + pSession, SESSIONS_PATCHSET, xOutput, pOut, 0, 0); } /* ** Obtain a patchset object containing all changes recorded by the ** session object passed as the first argument. @@ -3138,12 +3148,23 @@ sqlite3_session *pSession, /* Session object */ int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ void **ppPatchset /* OUT: Buffer containing changeset */ ){ if( pnPatchset==0 || ppPatchset==0 ) return SQLITE_MISUSE; - return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset); + return sessionGenerateChangeset( + pSession, SESSIONS_PATCHSET, 0, 0, pnPatchset, ppPatchset); } + +int sqlite3session_fullchangeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +){ + return sessionGenerateChangeset( + pSession, SESSIONS_FULLCHANGESET, 0, 0, pnChangeset, ppChangeset); +} + /* ** Enable or disable the session object passed as the first argument. */ int sqlite3session_enable(sqlite3_session *pSession, int bEnable){ @@ -5890,14 +5911,15 @@ /* Create the serialized output changeset based on the contents of the ** hash tables attached to the SessionTable objects in list p->pList. */ for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ + int eChangeset = pGrp->bPatch ? SESSIONS_PATCHSET : SESSIONS_CHANGESET; int i; if( pTab->nEntry==0 ) continue; - sessionAppendTableHdr(&buf, pGrp->bPatch, pTab, &rc); + sessionAppendTableHdr(&buf, eChangeset, pTab, &rc); for(i=0; inChange; i++){ SessionChange *p; for(p=pTab->apChange[i]; p; p=p->pNext){ sessionAppendByte(&buf, p->op, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); Index: ext/session/sqlite3session.h ================================================================== --- ext/session/sqlite3session.h +++ ext/session/sqlite3session.h @@ -361,10 +361,23 @@ ** Or, if one field of a row is updated while a session is disabled, and ** another field of the same row is updated while the session is enabled, the ** resulting changeset will contain an UPDATE change that updates both fields. */ int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Generate A Full Changeset From A Session Object +** +** This function is similar to sqlite3session_changeset(), except that for +** each row affected by an UPDATE statement, all old.* values are recorded +** as part of the changeset, not just those modified. +*/ +int sqlite3session_fullchangeset( sqlite3_session *pSession, /* Session object */ int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ void **ppChangeset /* OUT: Buffer containing changeset */ ); Index: ext/session/test_session.c ================================================================== --- ext/session/test_session.c +++ ext/session/test_session.c @@ -234,10 +234,11 @@ ** $session delete ** $session enable BOOL ** $session indirect INTEGER ** $session patchset ** $session table_filter SCRIPT +** $session fullchangeset */ static int SQLITE_TCLAPI test_session_cmd( void *clientData, Tcl_Interp *interp, int objc, @@ -247,24 +248,24 @@ sqlite3_session *pSession = p->pSession; static struct SessionSubcmd { const char *zSub; int nArg; const char *zMsg; - int iSub; } aSub[] = { - { "attach", 1, "TABLE", }, /* 0 */ - { "changeset", 0, "", }, /* 1 */ - { "delete", 0, "", }, /* 2 */ - { "enable", 1, "BOOL", }, /* 3 */ - { "indirect", 1, "BOOL", }, /* 4 */ - { "isempty", 0, "", }, /* 5 */ - { "table_filter", 1, "SCRIPT", }, /* 6 */ + { "attach", 1, "TABLE" }, /* 0 */ + { "changeset", 0, "" }, /* 1 */ + { "delete", 0, "" }, /* 2 */ + { "enable", 1, "BOOL" }, /* 3 */ + { "indirect", 1, "BOOL" }, /* 4 */ + { "isempty", 0, "" }, /* 5 */ + { "table_filter", 1, "SCRIPT" }, /* 6 */ { "patchset", 0, "", }, /* 7 */ - { "diff", 2, "FROMDB TBL", }, /* 8 */ - { "memory_used", 0, "", }, /* 9 */ - { "changeset_size", 0, "", }, /* 10 */ - { "object_config", 2, "OPTION INTEGER", }, /* 11 */ + { "diff", 2, "FROMDB TBL" }, /* 8 */ + { "fullchangeset",0, "" }, /* 9 */ + { "memory_used", 0, "", }, /* 10 */ + { "changeset_size", 0, "", }, /* 11 */ + { "object_config", 2, "OPTION INTEGER", }, /* 12 */ { 0 } }; int iSub; int rc; @@ -290,23 +291,26 @@ return test_session_error(interp, rc, 0); } break; } + case 9: /* fullchangeset */ case 7: /* patchset */ case 1: { /* changeset */ TestSessionsBlob o = {0, 0}; - if( test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ + if( iSub!=9 && test_tcl_integer(interp, SESSION_STREAM_TCL_VAR) ){ void *pCtx = (void*)&o; if( iSub==7 ){ rc = sqlite3session_patchset_strm(pSession, testStreamOutput, pCtx); }else{ rc = sqlite3session_changeset_strm(pSession, testStreamOutput, pCtx); } }else{ if( iSub==7 ){ rc = sqlite3session_patchset(pSession, &o.n, &o.p); + }else if( iSub==9 ){ + rc = sqlite3session_fullchangeset(pSession, &o.n, &o.p); }else{ rc = sqlite3session_changeset(pSession, &o.n, &o.p); } } if( rc==SQLITE_OK ){ @@ -317,10 +321,11 @@ if( rc!=SQLITE_OK ){ return test_session_error(interp, rc, 0); } break; } + case 2: /* delete */ Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); break; @@ -368,32 +373,31 @@ return test_session_error(interp, rc, zErr); } break; } - case 9: { /* memory_used */ + case 10: { /* memory_used */ sqlite3_int64 nMalloc = sqlite3session_memory_used(pSession); Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nMalloc)); break; } - case 10: { + case 11: { sqlite3_int64 nSize = sqlite3session_changeset_size(pSession); Tcl_SetObjResult(interp, Tcl_NewWideIntObj(nSize)); break; } - case 11: { /* object_config */ + case 12: { /* object_config */ struct ObjConfOpt { const char *zName; int opt; } aOpt[] = { { "size", SQLITE_SESSION_OBJCONFIG_SIZE }, { "rowid", SQLITE_SESSION_OBJCONFIG_ROWID }, { 0, 0 } }; size_t sz = sizeof(aOpt[0]); - int iArg; int iOpt; if( Tcl_GetIndexFromObjStruct(interp,objv[2],aOpt,sz,"option",0,&iOpt) ){ return TCL_ERROR; } @@ -546,11 +550,11 @@ Tcl_BackgroundError(interp); } Tcl_DecrRefCount(pEval); return res; -} +} static int test_conflict_handler( void *pCtx, /* Pointer to TestConflictHandler structure */ int eConf, /* DATA, MISSING, CONFLICT, CONSTRAINT */ sqlite3_changeset_iter *pIter /* Handle describing change and conflict */ @@ -1191,10 +1195,131 @@ return test_session_error(interp, rc, 0); } return TCL_OK; } + +#include "sqlite3changebatch.h" + +typedef struct TestChangebatch TestChangebatch; +struct TestChangebatch { + sqlite3_changebatch *pChangebatch; +}; + +/* +** Tclcmd: $changebatch add BLOB +** $changebatch zero +** $changebatch delete +*/ +static int SQLITE_TCLAPI test_changebatch_cmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + TestChangebatch *p = (TestChangebatch*)clientData; + sqlite3_changebatch *pChangebatch = p->pChangebatch; + struct SessionSubcmd { + const char *zSub; + int nArg; + const char *zMsg; + int iSub; + } aSub[] = { + { "add", 1, "CHANGESET", }, /* 0 */ + { "zero", 0, "", }, /* 1 */ + { "delete", 0, "", }, /* 2 */ + { 0 } + }; + int iSub; + int rc; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( objc!=2+aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: { /* add */ + int nArg; + unsigned char *pArg = Tcl_GetByteArrayFromObj(objv[2], &nArg); + rc = sqlite3changebatch_add(pChangebatch, pArg, nArg); + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ + return test_session_error(interp, rc, 0); + }else{ + extern const char *sqlite3ErrName(int); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + } + break; + } + + case 1: { /* zero */ + sqlite3changebatch_zero(pChangebatch); + break; + } + + case 2: /* delete */ + Tcl_DeleteCommand(interp, Tcl_GetString(objv[0])); + break; + } + + return TCL_OK; +} + +static void SQLITE_TCLAPI test_changebatch_del(void *clientData){ + TestChangebatch *p = (TestChangebatch*)clientData; + sqlite3changebatch_delete(p->pChangebatch); + ckfree((char*)p); +} + +/* +** Tclcmd: sqlite3changebatch CMD DB-HANDLE +*/ +static int SQLITE_TCLAPI test_sqlite3changebatch( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + Tcl_CmdInfo info; + int rc; /* sqlite3session_create() return code */ + TestChangebatch *p; /* New wrapper object */ + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CMD DB-HANDLE"); + return TCL_ERROR; + } + + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(objv[2]), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(objv[2]), 0); + return TCL_ERROR; + } + db = *(sqlite3 **)info.objClientData; + + p = (TestChangebatch*)ckalloc(sizeof(TestChangebatch)); + memset(p, 0, sizeof(TestChangebatch)); + rc = sqlite3changebatch_new(db, &p->pChangebatch); + if( rc!=SQLITE_OK ){ + ckfree((char*)p); + return test_session_error(interp, rc, 0); + } + + Tcl_CreateObjCommand( + interp, Tcl_GetString(objv[1]), test_changebatch_cmd, (ClientData)p, + test_changebatch_del + ); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} /* ** tclcmd: CMD configure REBASE-BLOB ** tclcmd: CMD rebase CHANGESET ** tclcmd: CMD delete @@ -1609,9 +1734,13 @@ for(i=0; izCmd, p->xProc, 0, 0); } + + Tcl_CreateObjCommand( + interp, "sqlite3changebatch", test_sqlite3changebatch, 0, 0 + ); return TCL_OK; } #endif /* SQLITE_TEST && SQLITE_SESSION && SQLITE_PREUPDATE_HOOK */ ADDED ext/wasm/EXPORTED_FUNCTIONS.fiddle Index: ext/wasm/EXPORTED_FUNCTIONS.fiddle ================================================================== --- /dev/null +++ ext/wasm/EXPORTED_FUNCTIONS.fiddle @@ -0,0 +1,7 @@ +_fiddle_exec +_fiddle_interrupt +_fiddle_experiment +_fiddle_the_db +_fiddle_db_arg +_fiddle_db_filename +_fiddle_reset_db DELETED ext/wasm/EXPORTED_FUNCTIONS.fiddle.in Index: ext/wasm/EXPORTED_FUNCTIONS.fiddle.in ================================================================== --- ext/wasm/EXPORTED_FUNCTIONS.fiddle.in +++ /dev/null @@ -1,10 +0,0 @@ -_fiddle_db_arg -_fiddle_db_filename -_fiddle_exec -_fiddle_experiment -_fiddle_interrupt -_fiddle_main -_fiddle_reset_db -_fiddle_db_handle -_fiddle_db_vfs -_fiddle_export_db Index: ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api ================================================================== --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -1,27 +1,17 @@ -_malloc -_free -_realloc -_sqlite3_aggregate_context -_sqlite3_auto_extension _sqlite3_bind_blob _sqlite3_bind_double _sqlite3_bind_int _sqlite3_bind_int64 _sqlite3_bind_null _sqlite3_bind_parameter_count _sqlite3_bind_parameter_index -_sqlite3_bind_pointer _sqlite3_bind_text -_sqlite3_busy_handler -_sqlite3_busy_timeout -_sqlite3_cancel_auto_extension _sqlite3_changes _sqlite3_changes64 _sqlite3_clear_bindings _sqlite3_close_v2 -_sqlite3_collation_needed _sqlite3_column_blob _sqlite3_column_bytes _sqlite3_column_count _sqlite3_column_count _sqlite3_column_double @@ -28,177 +18,55 @@ _sqlite3_column_int _sqlite3_column_int64 _sqlite3_column_name _sqlite3_column_text _sqlite3_column_type -_sqlite3_column_value -_sqlite3_commit_hook _sqlite3_compileoption_get _sqlite3_compileoption_used -_sqlite3_complete -_sqlite3_context_db_handle -_sqlite3_create_collation -_sqlite3_create_collation_v2 -_sqlite3_create_function _sqlite3_create_function_v2 -_sqlite3_create_module -_sqlite3_create_module_v2 -_sqlite3_create_window_function _sqlite3_data_count _sqlite3_db_filename -_sqlite3_db_handle _sqlite3_db_name -_sqlite3_db_status -_sqlite3_declare_vtab -_sqlite3_deserialize -_sqlite3_drop_modules -_sqlite3_errcode _sqlite3_errmsg _sqlite3_error_offset _sqlite3_errstr _sqlite3_exec _sqlite3_expanded_sql _sqlite3_extended_errcode _sqlite3_extended_result_codes -_sqlite3_file_control _sqlite3_finalize -_sqlite3_free -_sqlite3_get_auxdata -_sqlite3_get_autocommit _sqlite3_initialize -_sqlite3_keyword_count -_sqlite3_keyword_name -_sqlite3_keyword_check -_sqlite3_last_insert_rowid +_sqlite3_interrupt _sqlite3_libversion _sqlite3_libversion_number -_sqlite3_limit -_sqlite3_malloc -_sqlite3_malloc64 -_sqlite3_msize _sqlite3_open _sqlite3_open_v2 -_sqlite3_overload_function _sqlite3_prepare_v2 _sqlite3_prepare_v3 -_sqlite3_preupdate_blobwrite -_sqlite3_preupdate_count -_sqlite3_preupdate_depth -_sqlite3_preupdate_hook -_sqlite3_preupdate_new -_sqlite3_preupdate_old -_sqlite3_progress_handler -_sqlite3_randomness -_sqlite3_realloc -_sqlite3_realloc64 _sqlite3_reset -_sqlite3_reset_auto_extension _sqlite3_result_blob _sqlite3_result_double _sqlite3_result_error _sqlite3_result_error_code _sqlite3_result_error_nomem _sqlite3_result_error_toobig _sqlite3_result_int -_sqlite3_result_int64 _sqlite3_result_null -_sqlite3_result_pointer -_sqlite3_result_subtype _sqlite3_result_text -_sqlite3_result_zeroblob -_sqlite3_result_zeroblob64 -_sqlite3_rollback_hook -_sqlite3_serialize -_sqlite3_set_authorizer -_sqlite3_set_auxdata -_sqlite3_set_last_insert_rowid -_sqlite3_shutdown _sqlite3_sourceid _sqlite3_sql -_sqlite3_status -_sqlite3_status64 _sqlite3_step -_sqlite3_stmt_isexplain -_sqlite3_stmt_readonly -_sqlite3_stmt_status _sqlite3_strglob -_sqlite3_stricmp _sqlite3_strlike -_sqlite3_strnicmp -_sqlite3_table_column_metadata _sqlite3_total_changes _sqlite3_total_changes64 -_sqlite3_trace_v2 -_sqlite3_txn_state -_sqlite3_update_hook -_sqlite3_uri_boolean -_sqlite3_uri_int64 -_sqlite3_uri_key -_sqlite3_uri_parameter -_sqlite3_user_data _sqlite3_value_blob _sqlite3_value_bytes _sqlite3_value_double -_sqlite3_value_dup -_sqlite3_value_free -_sqlite3_value_frombind -_sqlite3_value_int -_sqlite3_value_int64 -_sqlite3_value_nochange -_sqlite3_value_numeric_type -_sqlite3_value_pointer -_sqlite3_value_subtype _sqlite3_value_text _sqlite3_value_type _sqlite3_vfs_find _sqlite3_vfs_register -_sqlite3_vfs_unregister -_sqlite3_vtab_collation -_sqlite3_vtab_distinct -_sqlite3_vtab_in -_sqlite3_vtab_in_first -_sqlite3_vtab_in_next -_sqlite3_vtab_nochange -_sqlite3_vtab_on_conflict -_sqlite3_vtab_rhs_value -_sqlite3changegroup_add -_sqlite3changegroup_add_strm -_sqlite3changegroup_delete -_sqlite3changegroup_new -_sqlite3changegroup_output -_sqlite3changegroup_output_strm -_sqlite3changeset_apply -_sqlite3changeset_apply_strm -_sqlite3changeset_apply_v2 -_sqlite3changeset_apply_v2_strm -_sqlite3changeset_concat -_sqlite3changeset_concat_strm -_sqlite3changeset_conflict -_sqlite3changeset_finalize -_sqlite3changeset_fk_conflicts -_sqlite3changeset_invert -_sqlite3changeset_invert_strm -_sqlite3changeset_new -_sqlite3changeset_next -_sqlite3changeset_old -_sqlite3changeset_op -_sqlite3changeset_pk -_sqlite3changeset_start -_sqlite3changeset_start_strm -_sqlite3changeset_start_v2 -_sqlite3changeset_start_v2_strm -_sqlite3session_attach -_sqlite3session_changeset -_sqlite3session_changeset_size -_sqlite3session_changeset_strm -_sqlite3session_config -_sqlite3session_create -_sqlite3session_delete -_sqlite3session_diff -_sqlite3session_enable -_sqlite3session_indirect -_sqlite3session_isempty -_sqlite3session_memory_used -_sqlite3session_object_config -_sqlite3session_patchset -_sqlite3session_patchset_strm -_sqlite3session_table_filter +_sqlite3_wasm_db_error +_sqlite3_wasm_enum_json +_malloc +_free Index: ext/wasm/api/sqlite3-worker1-promiser.c-pp.js ================================================================== --- ext/wasm/api/sqlite3-worker1-promiser.c-pp.js +++ ext/wasm/api/sqlite3-worker1-promiser.c-pp.js @@ -1,6 +1,5 @@ -//#ifnot omit-oo1 /* 2022-08-24 The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: @@ -40,17 +39,13 @@ function, enabling delayed instantiation of a Worker. - `onready` (optional, but...): this callback is called with no arguments when the worker fires its initial 'sqlite3-api'/'worker1-ready' message, which it does when - sqlite3.initWorker1API() completes its initialization. This is the - simplest way to tell the worker to kick off work at the earliest - opportunity, and the only way to know when the worker module has - completed loading. The irony of using a callback for this, instead - of returning a promise from sqlite3Worker1Promiser() is not lost on - the developers: see sqlite3Worker1Promiser.v2() which uses a - Promise instead. + sqlite3.initWorker1API() completes its initialization. This is + the simplest way to tell the worker to kick off work at the + earliest opportunity. - `onunhandled` (optional): a callback which gets passed the message event object for any worker.onmessage() events which are not handled by this proxy. Ideally that "should" never happen, as this proxy aims to handle all known message types. @@ -117,29 +112,20 @@ Where `typeString` is an internally-synthesized message type string used temporarily for worker message dispatching. It can be ignored by all client code except that which tests this API. The `row` property contains the row result in the form implied by the `rowMode` option (defaulting to `'array'`). The `rowNumber` is a - 1-based integer value incremented by 1 on each call into the + 1-based integer value incremented by 1 on each call into th callback. At the end of the result set, the same event is fired with (row=undefined, rowNumber=null) to indicate that the end of the result set has been reached. Note that the rows arrive via worker-posted messages, with all the implications of that. - - Notable shortcomings: - - - This API was not designed with ES6 modules in mind. Neither Firefox - nor Safari support, as of March 2023, the {type:"module"} flag to the - Worker constructor, so that particular usage is not something we're going - to target for the time being: - - https://developer.mozilla.org/en-US/docs/Web/API/Worker/Worker */ -globalThis.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ +self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ // Inspired by: https://stackoverflow.com/a/52439530 if(1===arguments.length && 'function'===typeof arguments[0]){ const f = config; config = Object.assign(Object.create(null), callee.defaultConfig); config.onready = f; @@ -158,26 +144,25 @@ }; const toss = (...args)=>{throw new Error(args.join(' '))}; if(!config.worker) config.worker = callee.defaultConfig.worker; if('function'===typeof config.worker) config.worker = config.worker(); let dbId; - let promiserFunc; config.worker.onmessage = function(ev){ ev = ev.data; debug('worker1.onmessage',ev); let msgHandler = handlerMap[ev.messageId]; if(!msgHandler){ if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { /*fired one time when the Worker1 API initializes*/ - if(config.onready) config.onready(promiserFunc); + if(config.onready) config.onready(); return; } msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; if(msgHandler && msgHandler.onrow){ msgHandler.onrow(ev); return; - } + } if(config.onunhandled) config.onunhandled(arguments[0]); else err("sqlite3Worker1Promiser() unhandled worker message:",ev); return; } delete handlerMap[ev.messageId]; @@ -195,23 +180,23 @@ break; } try {msgHandler.resolve(ev)} catch(e){msgHandler.reject(e)} }/*worker.onmessage()*/; - return promiserFunc = function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ let msg; if(1===arguments.length){ msg = arguments[0]; }else if(2===arguments.length){ - msg = Object.create(null); - msg.type = arguments[0]; - msg.args = arguments[1]; - msg.dbId = msg.args.dbId; + msg = { + type: arguments[0], + args: arguments[1] + }; }else{ toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); } - if(!msg.dbId && msg.type!=='open') msg.dbId = dbId; + if(!msg.dbId) msg.dbId = dbId; msg.messageId = genMsgId(msg); msg.departureTime = performance.now(); const proxy = Object.create(null); proxy.message = msg; let rowCallbackId /* message handler ID for exec on-row callback proxy */; @@ -249,98 +234,30 @@ }); if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]); return p; }; }/*sqlite3Worker1Promiser()*/; - -globalThis.sqlite3Worker1Promiser.defaultConfig = { +self.sqlite3Worker1Promiser.defaultConfig = { worker: function(){ -//#if target=es6-module - return new Worker(new URL("sqlite3-worker1-bundler-friendly.mjs", import.meta.url),{ - type: 'module' - }); +//#if target=es6-bundler-friendly + return new Worker("sqlite3-worker1.js"); //#else let theJs = "sqlite3-worker1.js"; if(this.currentScript){ const src = this.currentScript.src.split('/'); src.pop(); theJs = src.join('/')+'/' + theJs; //sqlite3.config.warn("promiser currentScript, theJs =",this.currentScript,theJs); - }else if(globalThis.location){ - //sqlite3.config.warn("promiser globalThis.location =",globalThis.location); - const urlParams = new URL(globalThis.location.href).searchParams; + }else{ + //sqlite3.config.warn("promiser self.location =",self.location); + const urlParams = new URL(self.location.href).searchParams; if(urlParams.has('sqlite3.dir')){ theJs = urlParams.get('sqlite3.dir') + '/' + theJs; } } - return new Worker(theJs + globalThis.location.search); -//#endif - } -//#ifnot target=es6-module - .bind({ - currentScript: globalThis?.document?.currentScript - }) -//#endif - , + return new Worker(theJs + self.location.search); +//#endif + }.bind({ + currentScript: self?.document?.currentScript + }), onerror: (...args)=>console.error('worker1 promiser error',...args) -}/*defaultConfig*/; - -/** - sqlite3Worker1Promiser.v2() works identically to - sqlite3Worker1Promiser() except that it returns a Promise instead - of relying an an onready callback in the config object. The Promise - resolves to the same factory function which - sqlite3Worker1Promiser() returns. - - If config is-a function or is an object which contains an onready - function, that function is replaced by a proxy which will resolve - after calling the original function and will reject if that - function throws. -*/ -sqlite3Worker1Promiser.v2 = function(config){ - let oldFunc; - if( 'function' == typeof config ){ - oldFunc = config; - config = {}; - }else if('function'===typeof config?.onready){ - oldFunc = config.onready; - delete config.onready; - } - const promiseProxy = Object.create(null); - config = Object.assign((config || Object.create(null)),{ - onready: async function(func){ - try { - if( oldFunc ) await oldFunc(func); - promiseProxy.resolve(func); - } - catch(e){promiseProxy.reject(e)} - } - }); - const p = new Promise(function(resolve,reject){ - promiseProxy.resolve = resolve; - promiseProxy.reject = reject; - }); - try{ - this.original(config); - }catch(e){ - promiseProxy.reject(e); - } - return p; -}.bind({ - /* We do this because clients are - recommended to delete globalThis.sqlite3Worker1Promiser. */ - original: sqlite3Worker1Promiser -}); - -//#if target=es6-module -/** - When built as a module, we export sqlite3Worker1Promiser.v2() - instead of sqlite3Worker1Promise() because (A) its interface is more - conventional for ESM usage and (B) the ESM option export option for - this API did not exist until v2 was created, so there's no backwards - incompatibility. -*/ -export default sqlite3Worker1Promiser.v2; -//#endif /* target=es6-module */ -//#else -/* Built with the omit-oo1 flag. */ -//#endif ifnot omit-oo1 +}; ADDED ext/wasm/fiddle/fiddle.html Index: ext/wasm/fiddle/fiddle.html ================================================================== --- /dev/null +++ ext/wasm/fiddle/fiddle.html @@ -0,0 +1,283 @@ + + + + + + SQLite3 Fiddle + + + + + + + +
+ SQLite3 Fiddle + Powered by + SQLite3 +
+ +
+
+
Initializing app...
+
+ On a slow internet connection this may take a moment. If this + message displays for "a long time", intialization may have + failed and the JavaScript console may contain clues as to why. +
+
+
Downloading...
+
+ +
+ + + +
+
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+ + + + +
+
+
+
+
+ + + +
+
+
+
+
+ + + + DELETED ext/wasm/fiddle/index.html Index: ext/wasm/fiddle/index.html ================================================================== --- ext/wasm/fiddle/index.html +++ /dev/null @@ -1,278 +0,0 @@ - - - - - - SQLite3 Fiddle - - - - - - - -
- SQLite3 Fiddle - Powered by - SQLite3 -
- -
-
-
Initializing app...
-
- On a slow internet connection this may take a moment. If this - message displays for "a long time", intialization may have - failed and the JavaScript console may contain clues as to why. -
-
-
Downloading...
-
- -
- - - -
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
- - - - -
-
-
-
-
- - - -
-
-
-
-
- - - Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -428,10 +428,11 @@ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c \ $(TOP)/ext/misc/stmt.c \ $(TOP)/ext/session/sqlite3session.c \ + $(TOP)/ext/session/sqlite3changebatch.c \ $(TOP)/ext/session/test_session.c \ fts5.c # Header files used by all library source files. # @@ -1003,10 +1004,13 @@ $(TOP)/test/tt3_stress.c \ $(TOP)/test/tt3_lookaside1.c threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) $(TOP)/src/test_multiplex.c $(TCCX) $(TOP)/test/threadtest3.c $(TOP)/src/test_multiplex.c sqlite3.o -o $@ $(THREADLIB) + +bc_test1$(EXE): sqlite3.o $(TOP)/test/bc_test1.c $(TOP)/test/tt3_core.c + $(TCCX) $(TOP)/test/bc_test1.c sqlite3.o -o $@ $(THREADLIB) threadtest: threadtest3$(EXE) ./threadtest3$(EXE) TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO) Index: src/bitvec.c ================================================================== --- src/bitvec.c +++ src/bitvec.c @@ -169,10 +169,16 @@ int sqlite3BitvecSet(Bitvec *p, u32 i){ u32 h; if( p==0 ) return SQLITE_OK; assert( i>0 ); assert( i<=p->iSize ); + if( i>p->iSize || i==0 ){ + sqlite3_log(SQLITE_ERROR, + "Bitvec: setting bit %d of bitvec size %d\n", (int)i, (int)p->iSize + ); + abort(); + } i--; while((p->iSize > BITVEC_NBIT) && p->iDivisor) { u32 bin = i/p->iDivisor; i = i%p->iDivisor; if( p->u.apSub[bin]==0 ){ Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -526,11 +526,244 @@ } } #endif /* SQLITE_OMIT_SHARED_CACHE */ -static void releasePage(MemPage *pPage); /* Forward reference */ +#ifndef SQLITE_OMIT_CONCURRENT +/* +** The following structure - BtreePtrmap - stores the in-memory pointer map +** used for newly allocated pages in CONCURRENT transactions. Such pages are +** always allocated in a contiguous block (from the end of the file) starting +** with page BtreePtrmap.iFirst. +*/ +typedef struct RollbackEntry RollbackEntry; +typedef struct PtrmapEntry PtrmapEntry; +struct PtrmapEntry { + Pgno parent; + u8 eType; +}; +struct RollbackEntry { + Pgno pgno; + Pgno parent; + u8 eType; +}; +struct BtreePtrmap { + Pgno iFirst; /* First new page number aPtr[0] */ + + int nPtrAlloc; /* Allocated size of aPtr[] array */ + PtrmapEntry *aPtr; /* Array of parent page numbers */ + + int nSvpt; /* Used size of aSvpt[] array */ + int nSvptAlloc; /* Allocated size of aSvpt[] */ + int *aSvpt; /* First aRollback[] entry for savepoint i */ + + int nRollback; /* Used size of aRollback[] array */ + int nRollbackAlloc; /* Allocated size of aRollback[] array */ + RollbackEntry *aRollback; /* Array of rollback entries */ +}; + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** If page number pgno is greater than or equal to BtreePtrmap.iFirst, +** store an entry for it in the pointer-map structure. +*/ +static int btreePtrmapStore( + BtShared *pBt, + Pgno pgno, + u8 eType, + Pgno parent +){ + BtreePtrmap *pMap = pBt->pMap; + if( pgno>=pMap->iFirst ){ + int iEntry = pgno - pMap->iFirst; + + /* Grow the aPtr[] array as required */ + while( iEntry>=pMap->nPtrAlloc ){ + int nNew = pMap->nPtrAlloc ? pMap->nPtrAlloc*2 : 16; + PtrmapEntry *aNew = (PtrmapEntry*)sqlite3_realloc( + pMap->aPtr, nNew*sizeof(PtrmapEntry) + ); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + int nByte = (nNew-pMap->nPtrAlloc)*sizeof(PtrmapEntry); + memset(&aNew[pMap->nPtrAlloc], 0, nByte); + pMap->aPtr = aNew; + pMap->nPtrAlloc = nNew; + } + } + + /* Add an entry to the rollback log if required */ + if( pMap->nSvpt>0 && pMap->aPtr[iEntry].parent ){ + if( pMap->nRollback>=pMap->nRollbackAlloc ){ + int nNew = pMap->nRollback ? pMap->nRollback*2 : 16; + RollbackEntry *aNew = (RollbackEntry*)sqlite3_realloc( + pMap->aRollback, nNew*sizeof(RollbackEntry) + ); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + pMap->aRollback = aNew; + pMap->nRollbackAlloc = nNew; + } + } + + pMap->aRollback[pMap->nRollback].pgno = pgno; + pMap->aRollback[pMap->nRollback].parent = pMap->aPtr[iEntry].parent; + pMap->aRollback[pMap->nRollback].eType = pMap->aPtr[iEntry].eType; + pMap->nRollback++; + } + + /* Update the aPtr[] array */ + pMap->aPtr[iEntry].parent = parent; + pMap->aPtr[iEntry].eType = eType; + } + + return SQLITE_OK; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Open savepoint iSavepoint, if it is not already open. +*/ +static int btreePtrmapBegin(BtShared *pBt, int nSvpt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap && nSvpt>pMap->nSvpt ){ + int i; + if( nSvpt>=pMap->nSvptAlloc ){ + int nNew = pMap->nSvptAlloc ? pMap->nSvptAlloc*2 : 16; + int *aNew = sqlite3_realloc(pMap->aSvpt, sizeof(int) * nNew); + if( aNew==0 ){ + return SQLITE_NOMEM; + }else{ + pMap->aSvpt = aNew; + pMap->nSvptAlloc = nNew; + } + } + + for(i=pMap->nSvpt; iaSvpt[i] = pMap->nRollback; + } + pMap->nSvpt = nSvpt; + } + + return SQLITE_OK; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Rollback (if op==SAVEPOINT_ROLLBACK) or release (if op==SAVEPOINT_RELEASE) +** savepoint iSvpt. +*/ +static void btreePtrmapEnd(BtShared *pBt, int op, int iSvpt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap ){ + assert( op==SAVEPOINT_ROLLBACK || op==SAVEPOINT_RELEASE ); + assert( iSvpt>=0 || (iSvpt==-1 && op==SAVEPOINT_ROLLBACK) ); + if( iSvpt<0 ){ + pMap->nSvpt = 0; + pMap->nRollback = 0; + memset(pMap->aPtr, 0, sizeof(Pgno) * pMap->nPtrAlloc); + }else if( iSvptnSvpt ){ + if( op==SAVEPOINT_ROLLBACK ){ + int ii; + for(ii=pMap->nRollback-1; ii>=pMap->aSvpt[iSvpt]; ii--){ + RollbackEntry *p = &pMap->aRollback[ii]; + PtrmapEntry *pEntry = &pMap->aPtr[p->pgno - pMap->iFirst]; + pEntry->parent = p->parent; + pEntry->eType = p->eType; + } + } + pMap->nSvpt = iSvpt + (op==SAVEPOINT_ROLLBACK); + pMap->nRollback = pMap->aSvpt[iSvpt]; + } + } +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** This function is called after an CONCURRENT transaction is opened on the +** database. It allocates the BtreePtrmap structure used to track pointers +** to allocated pages and zeroes the nFree/iTrunk fields in the database +** header on page 1. +*/ +static int btreePtrmapAllocate(BtShared *pBt){ + int rc = SQLITE_OK; + if( pBt->pMap==0 ){ + BtreePtrmap *pMap = sqlite3_malloc(sizeof(BtreePtrmap)); + if( pMap==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&pBt->pPage1->aData[32], 0, sizeof(u32)*2); + memset(pMap, 0, sizeof(BtreePtrmap)); + pMap->iFirst = pBt->nPage + 1; + pBt->pMap = pMap; + } + } + return rc; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Free any BtreePtrmap structure allocated by an earlier call to +** btreePtrmapAllocate(). +*/ +static void btreePtrmapDelete(BtShared *pBt){ + BtreePtrmap *pMap = pBt->pMap; + if( pMap ){ + sqlite3_free(pMap->aRollback); + sqlite3_free(pMap->aPtr); + sqlite3_free(pMap->aSvpt); + sqlite3_free(pMap); + pBt->pMap = 0; + } +} + +/* +** Check that the pointer-map does not contain any entries with a parent +** page of 0. Call sqlite3_log() multiple times to output the entire +** data structure if it does. +*/ +static void btreePtrmapCheck(BtShared *pBt, Pgno nPage){ + Pgno i; + int bProblem = 0; + BtreePtrmap *p = pBt->pMap; + + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + if( pEntry->eType==PTRMAP_OVERFLOW1 + || pEntry->eType==PTRMAP_OVERFLOW2 + || pEntry->eType==PTRMAP_BTREE + ){ + if( pEntry->parent==0 ){ + bProblem = 1; + break; + } + } + } + + if( bProblem ){ + for(i=p->iFirst; i<=nPage; i++){ + PtrmapEntry *pEntry = &p->aPtr[i-p->iFirst]; + sqlite3_log(SQLITE_CORRUPT, + "btreePtrmapCheck: pgno=%d eType=%d parent=%d", + (int)i, (int)pEntry->eType, (int)pEntry->parent + ); + } + abort(); + } +} + +#else /* SQLITE_OMIT_CONCURRENT */ +# define btreePtrmapAllocate(x) SQLITE_OK +# define btreePtrmapDelete(x) +# define btreePtrmapBegin(x,y) SQLITE_OK +# define btreePtrmapEnd(x,y,z) +# define btreePtrmapCheck(y,z) +#endif /* SQLITE_OMIT_CONCURRENT */ + +static void releasePage(MemPage *pPage); /* Forward reference */ static void releasePageOne(MemPage *pPage); /* Forward reference */ static void releasePageNotNull(MemPage *pPage); /* Forward reference */ /* ***** This routine is used inside of assert() only **** @@ -1067,10 +1300,17 @@ if( *pRC ) return; assert( sqlite3_mutex_held(pBt->mutex) ); /* The super-journal page number must never be used as a pointer map page */ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + +#ifndef SQLITE_OMIT_CONCURRENT + if( pBt->pMap ){ + *pRC = btreePtrmapStore(pBt, key, eType, parent); + return; + } +#endif assert( pBt->autoVacuum ); if( key==0 ){ *pRC = SQLITE_CORRUPT_BKPT; return; @@ -2400,10 +2640,21 @@ assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); *ppPage = pPage; return SQLITE_OK; } +#ifndef SQLITE_OMIT_CONCURRENT +/* +** Set the value of the MemPage.pgnoRoot variable, if it exists. +*/ +static void setMempageRoot(MemPage *pPg, u32 pgnoRoot){ + pPg->pgnoRoot = pgnoRoot; +} +#else +# define setMempageRoot(x,y) +#endif + /* ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. ** ** Page1 is a special case and must be released using releasePageOne(). @@ -3586,10 +3837,11 @@ int *pSchemaVersion /* Put schema version number here, if not NULL */ ){ BtShared *pBt = p->pBt; Pager *pPager = pBt->pPager; int rc = SQLITE_OK; + int bConcurrent = (p->db->eConcurrent && !ISAUTOVACUUM(pBt)); sqlite3BtreeEnter(p); btreeIntegrity(p); /* If the btree is already in a write-transaction, or it @@ -3673,11 +3925,12 @@ if( rc==SQLITE_OK && wrflag ){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); + int exFlag = bConcurrent ? -1 : (wrflag>1); + rc = sqlite3PagerBegin(pPager, exFlag, sqlite3TempInMemory(p->db)); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ /* if there was no transaction opened when this function was ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error @@ -3737,20 +3990,33 @@ } } } trans_begun: +#ifndef SQLITE_OMIT_CONCURRENT + if( bConcurrent && rc==SQLITE_OK && sqlite3PagerIsWal(pBt->pPager) ){ + rc = sqlite3PagerBeginConcurrent(pBt->pPager); + if( rc==SQLITE_OK && wrflag ){ + rc = btreePtrmapAllocate(pBt); + } + } +#endif + if( rc==SQLITE_OK ){ if( pSchemaVersion ){ *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); } if( wrflag ){ /* This call makes sure that the pager has the correct number of ** open savepoints. If the second parameter is greater than 0 and ** the sub-journal is not already open, then it will be opened here. */ - rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + int nSavepoint = p->db->nSavepoint; + rc = sqlite3PagerOpenSavepoint(pPager, nSavepoint); + if( rc==SQLITE_OK && nSavepoint ){ + rc = btreePtrmapBegin(pBt, nSavepoint); + } } } btreeIntegrity(p); sqlite3BtreeLeave(p); @@ -3759,10 +4025,19 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ BtShared *pBt; if( p->sharable || p->inTrans==TRANS_NONE || (p->inTrans==TRANS_READ && wrflag!=0) +#ifndef SQLITE_OMIT_CONCURRENT + /* Always use the full version for "BEGIN CONCURRENT" transactions. This + ** is to ensure that any required calls to btreePtrmapBegin() are made. + ** These calls are not present on trunk (they're part of the + ** begin-concurrent patch), and so they are not present in the fast path + ** below. And it's easier just to call the full version every time than + ** to complicate the code below by adding btreePtrmapBegin() calls. */ + || p->db->eConcurrent!=CONCURRENT_NONE +#endif ){ return btreeBeginTrans(p,wrflag,pSchemaVersion); } pBt = p->pBt; if( pSchemaVersion ){ @@ -4236,10 +4511,179 @@ #else /* ifndef SQLITE_OMIT_AUTOVACUUM */ # define setChildPtrmaps(x) SQLITE_OK #endif +#ifndef SQLITE_OMIT_CONCURRENT +/* +** This function is called as part of merging an CONCURRENT transaction with +** the snapshot at the head of the wal file. It relocates all pages in the +** range iFirst..iLast, inclusive. It is assumed that the BtreePtrmap +** structure at BtShared.pMap contains the location of the pointers to each +** page in the range. +** +** If pnCurrent is NULL, then all pages in the range are moved to currently +** free locations (i.e. free-list entries) within the database file before page +** iFirst. +** +** Or, if pnCurrent is not NULL, then it points to a value containing the +** current size of the database file in pages. In this case, all pages are +** relocated to the end of the database file - page iFirst is relocated to +** page (*pnCurrent+1), page iFirst+1 to page (*pnCurrent+2), and so on. +** Value *pnCurrent is set to the new size of the database before this +** function returns. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int btreeRelocateRange( + BtShared *pBt, /* B-tree handle */ + Pgno iFirst, /* First page to relocate */ + Pgno iLast, /* Last page to relocate */ + Pgno *pnCurrent /* If not NULL, IN/OUT: Database size */ +){ + int rc = SQLITE_OK; + BtreePtrmap *pMap = pBt->pMap; + Pgno iPg; + + for(iPg=iFirst; iPg<=iLast && rc==SQLITE_OK; iPg++){ + MemPage *pFree = 0; /* Page allocated from free-list */ + MemPage *pPg = 0; + Pgno iNew; /* New page number for pPg */ + PtrmapEntry *pEntry; /* Pointer map entry for page iPg */ + + if( iPg==PENDING_BYTE_PAGE(pBt) ) continue; + pEntry = &pMap->aPtr[iPg - pMap->iFirst]; + + if( pEntry->eType==PTRMAP_FREEPAGE ){ + Pgno dummy; + rc = allocateBtreePage(pBt, &pFree, &dummy, iPg, BTALLOC_EXACT); + if( pFree ){ + assert( sqlite3PagerPageRefcount(pFree->pDbPage)==1 ); + sqlite3PcacheDrop(pFree->pDbPage); + } + assert( rc!=SQLITE_OK || dummy==iPg ); + }else if( pnCurrent ){ + btreeGetPage(pBt, iPg, &pPg, 0); + assert( sqlite3PagerIswriteable(pPg->pDbPage) ); + assert( sqlite3PagerPageRefcount(pPg->pDbPage)==1 ); + iNew = ++(*pnCurrent); + if( iNew==PENDING_BYTE_PAGE(pBt) ) iNew = ++(*pnCurrent); + rc = relocatePage(pBt, pPg, pEntry->eType, pEntry->parent, iNew, 1); + releasePageNotNull(pPg); + }else{ + rc = allocateBtreePage(pBt, &pFree, &iNew, iFirst-1, BTALLOC_LE); + assert( rc!=SQLITE_OK || iNeweType, pEntry->parent,iNew,1); + releasePage(pPg); + } + } + } + return rc; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** The b-tree handle passed as the only argument is about to commit an +** CONCURRENT transaction. At this point it is guaranteed that this is +** possible - the wal WRITER lock is held and it is known that there are +** no conflicts with committed transactions. +*/ +static int btreeFixUnlocked(Btree *p){ + BtShared *pBt = p->pBt; + MemPage *pPage1 = pBt->pPage1; + u8 *p1 = pPage1->aData; + Pager *pPager = pBt->pPager; + int rc = SQLITE_OK; + + /* If page 1 of the database is not writable, then no pages were allocated + ** or freed by this transaction. In this case no special handling is + ** required. Otherwise, if page 1 is dirty, proceed. */ + BtreePtrmap *pMap = pBt->pMap; + Pgno iTrunk = get4byte(&p1[32]); + Pgno nPage = btreePagecount(pBt); + u32 nFree = get4byte(&p1[36]); + + assert( pBt->pMap ); + rc = sqlite3PagerUpgradeSnapshot(pPager, pPage1->pDbPage); + assert( p1==pPage1->aData ); + + if( rc==SQLITE_OK ){ + Pgno nHPage = get4byte(&p1[28]); + Pgno nFin = nHPage; /* Size of db after transaction merge */ + + if( sqlite3PagerIswriteable(pPage1->pDbPage) ){ + Pgno iHTrunk = get4byte(&p1[32]); + u32 nHFree = get4byte(&p1[36]); + + btreePtrmapCheck(pBt, nPage); + + /* Attach the head database free list to the end of the current + ** transactions free-list (if any). */ + if( iTrunk!=0 ){ + put4byte(&p1[36], nHFree + nFree); + put4byte(&p1[32], iTrunk); + while( iTrunk ){ + DbPage *pTrunk = sqlite3PagerLookup(pPager, iTrunk); + iTrunk = get4byte((u8*)pTrunk->pData); + if( iTrunk==0 ){ + put4byte((u8*)pTrunk->pData, iHTrunk); + } + sqlite3PagerUnref(pTrunk); + }; + } + + if( nHPage<(pMap->iFirst-1) ){ + /* The database consisted of (pMap->iFirst-1) pages when the current + ** concurrent transaction was opened. And an concurrent transaction may + ** not be executed on an auto-vacuum database - so the db should + ** not have shrunk since the transaction was opened. Therefore nHPage + ** should be set to (pMap->iFirst-1) or greater. */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + /* The current transaction allocated pages pMap->iFirst through + ** nPage (inclusive) at the end of the database file. Meanwhile, + ** other transactions have allocated (iFirst..nHPage). So move + ** pages (iFirst..MIN(nPage,nHPage)) to (MAX(nPage,nHPage)+1). */ + Pgno iLast = MIN(nPage, nHPage); /* Last page to move */ + Pgno nCurrent; /* Current size of db */ + + nCurrent = MAX(nPage, nHPage); + pBt->nPage = nCurrent; + rc = btreeRelocateRange(pBt, pMap->iFirst, iLast, &nCurrent); + + /* There are now no collisions with the snapshot at the head of the + ** database file. So at this point it would be possible to write + ** the transaction out to disk. Before doing so though, attempt to + ** relocate some of the new pages to free locations within the body + ** of the database file (i.e. free-list entries). */ + if( rc==SQLITE_OK ){ + assert( nCurrent!=PENDING_BYTE_PAGE(pBt) ); + sqlite3PagerSetDbsize(pBt->pPager, nCurrent); + nFree = get4byte(&p1[36]); + nFin = nCurrent-nFree; + if( nCurrent>PENDING_BYTE_PAGE(pBt) && nFin<=PENDING_BYTE_PAGE(pBt) ){ + nFin--; + } + nFin = MAX(nFin, nHPage); + rc = btreeRelocateRange(pBt, nFin+1, nCurrent, 0); + } + + put4byte(&p1[28], nFin); + } + } + sqlite3PagerSetDbsize(pPager, nFin); + } + + return rc; +} +#else +# define btreeFixUnlocked(X) SQLITE_OK +#endif /* SQLITE_OMIT_CONCURRENT */ + /* ** This routine does the first phase of a two-phase commit. This routine ** causes a rollback journal to be created (if it does not already exist) ** and populated with enough information so that if a power loss occurs ** the database can be restored to its original state by playing back @@ -4267,12 +4711,14 @@ int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ int rc = SQLITE_OK; if( p->inTrans==TRANS_WRITE ){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); + #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ + assert( ISCONCURRENT==0 ); rc = autoVacuumCommit(p); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); return rc; } @@ -4279,11 +4725,16 @@ } if( pBt->bDoTruncate ){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + if( rc==SQLITE_OK && ISCONCURRENT && p->db->eConcurrent==CONCURRENT_OPEN ){ + rc = btreeFixUnlocked(p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + } sqlite3BtreeLeave(p); } return rc; } @@ -4322,10 +4773,15 @@ ** pager if this call closed the only read or write transaction. */ p->inTrans = TRANS_NONE; unlockBtreeIfUnused(pBt); } + /* If this was an CONCURRENT transaction, delete the pBt->pMap object. + ** Also call PagerEndConcurrent() to ensure that the pager has discarded + ** the record of all pages read within the transaction. */ + btreePtrmapDelete(pBt); + sqlite3PagerEndConcurrent(pBt->pPager); btreeIntegrity(p); } /* ** Commit the transaction currently in progress. @@ -4551,10 +5007,13 @@ ** an index greater than all savepoints created explicitly using ** SQL statements. It is illegal to open, release or rollback any ** such savepoints while the statement transaction savepoint is active. */ rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + if( rc==SQLITE_OK ){ + rc = btreePtrmapBegin(pBt, iStatement); + } sqlite3BtreeLeave(p); return rc; } /* @@ -4574,10 +5033,11 @@ if( p && p->inTrans==TRANS_WRITE ){ BtShared *pBt = p->pBt; assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); + btreePtrmapEnd(pBt, op, iSavepoint); if( op==SAVEPOINT_ROLLBACK ){ rc = saveAllCursors(pBt, 0, 0); } if( rc==SQLITE_OK ){ rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); @@ -5213,10 +5673,12 @@ sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); u8 aSave[4]; u8 *aWrite = &pBuf[-4]; assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); + rc = sqlite3PagerUsePage(pBt->pPager, nextPage); + if( rc!=SQLITE_OK ) break; rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else #endif @@ -5225,10 +5687,13 @@ DbPage *pDbPage; rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, (eOp==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ + setMempageRoot( + (MemPage*)sqlite3PagerGetExtra(pDbPage), pCur->pgnoRoot + ); aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; @@ -5375,10 +5840,11 @@ ** the new child page does not match the flags field of the parent (i.e. ** if an intkey page appears to be the parent of a non-intkey page, or ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ + BtShared *pBt = pCur->pBt; int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); @@ -5389,17 +5855,18 @@ pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); pCur->aiIdx[pCur->iPage] = pCur->ix; pCur->apPage[pCur->iPage] = pCur->pPage; pCur->ix = 0; pCur->iPage++; - rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); - assert( pCur->pPage!=0 || rc!=SQLITE_OK ); - if( rc==SQLITE_OK - && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) - ){ - releasePage(pCur->pPage); - rc = SQLITE_CORRUPT_PGNO(newPgno); + rc = getAndInitPage(pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); + if( rc==SQLITE_OK ){ + assert( pCur->pPage!=0 ); + setMempageRoot(pCur->pPage, pCur->pgnoRoot); + if( pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(newPgno); + } } if( rc ){ pCur->pPage = pCur->apPage[--pCur->iPage]; } return rc; @@ -5512,10 +5979,11 @@ pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } + setMempageRoot(pCur->pPage, pCur->pgnoRoot); pCur->iPage = 0; pCur->curIntKey = pCur->pPage->intKey; } pRoot = pCur->pPage; assert( pRoot->pgno==pCur->pgnoRoot || CORRUPT_DB ); @@ -6154,10 +6622,11 @@ } if( rc ){ pCur->pPage = pCur->apPage[--pCur->iPage]; break; } + setMempageRoot(pCur->pPage, pCur->pgnoRoot); /* ***** End of in-lined moveToChild() call */ } moveto_index_finish: pCur->info.nSize = 0; @@ -6424,20 +6893,29 @@ MemPage *pTrunk = 0; MemPage *pPrevTrunk = 0; Pgno mxPage; /* Total size of the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); - assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && REQUIRE_PTRMAP ) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36 ** stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ return SQLITE_CORRUPT_BKPT; } + + /* Ensure page 1 is writable. This function will either change the number + ** of pages in the free-list or the size of the database file. Since both + ** of these operations involve modifying page 1 header fields, page 1 + ** will definitely be written by this transaction. If this is an CONCURRENT + ** transaction, ensure the BtreePtrmap structure has been allocated. */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + if( n>0 ){ /* There are pages on the freelist. Reuse one of those pages. */ Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ u32 nSearch = 0; /* Count of the number of search attempts */ @@ -6444,32 +6922,33 @@ /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. */ -#ifndef SQLITE_OMIT_AUTOVACUUM if( eMode==BTALLOC_EXACT ){ - if( nearby<=mxPage ){ - u8 eType; - assert( nearby>0 ); - assert( pBt->autoVacuum ); - rc = ptrmapGet(pBt, nearby, &eType, 0); - if( rc ) return rc; - if( eType==PTRMAP_FREEPAGE ){ - searchList = 1; - } + assert( ISAUTOVACUUM(pBt)!=ISCONCURRENT ); + if( ISAUTOVACUUM(pBt) ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + } + }else{ + searchList = 1; } }else if( eMode==BTALLOC_LE ){ searchList = 1; } -#endif /* Decrement the free-list count by 1. Set iTrunk to the index of the ** first free-list trunk page. iPrevTrunk is initially 1. */ - rc = sqlite3PagerWrite(pPage1->pDbPage); - if( rc ) return rc; put4byte(&pPage1->aData[36], n-1); /* The code within this loop is run only once if the 'searchList' variable ** is not true. Otherwise, it runs once for each trunk-page on the ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) @@ -6773,11 +7252,11 @@ } /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } /* Now manipulate the actual database free-list structure. There are two @@ -7107,11 +7586,11 @@ PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) ); } #endif rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); -#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map ** for that page now. ** ** If this is the first overflow page, then write a partial entry @@ -7118,18 +7597,17 @@ ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ - if( pBt->autoVacuum && rc==SQLITE_OK ){ + if( REQUIRE_PTRMAP && rc==SQLITE_OK ){ u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); if( rc ){ releasePage(pOvfl); } } -#endif if( rc ){ releasePage(pToRelease); return rc; } @@ -7269,10 +7747,11 @@ ** balancing, and the dividers are adjacent and sorted. */ assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ + BtShared *pBt = pPage->pBt; int rc = sqlite3PagerWrite(pPage->pDbPage); if( NEVER(rc!=SQLITE_OK) ){ return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -7298,20 +7777,18 @@ put2byte(pIns, idx); pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPage->pBt->autoVacuum ){ + if( REQUIRE_PTRMAP ){ int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); if( rc2 ) return rc2; } -#endif } return SQLITE_OK; } /* @@ -7359,10 +7836,11 @@ ** balancing, and the dividers are adjacent and sorted. */ assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ }else{ + BtShared *pBt = pPage->pBt; int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -7372,30 +7850,28 @@ if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); - assert( idx+sz <= (int)pPage->pBt->usableSize ); + assert( idx+sz <= (int)pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPage->pBt->autoVacuum ){ + if( REQUIRE_PTRMAP ){ int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); if( rc2 ) return rc2; } -#endif } return SQLITE_OK; } /* @@ -7947,11 +8423,11 @@ ** of the parent page are still manipulated by the code below. ** That is Ok, at this point the parent page is guaranteed to ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } } @@ -8085,11 +8561,11 @@ } /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ *pRC = setChildPtrmaps(pTo); } } } @@ -8136,11 +8612,12 @@ static int balance_nonroot( MemPage *pParent, /* Parent page of siblings being balanced */ int iParentIdx, /* Index of "the page" in pParent */ u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ int isRoot, /* True if pParent is a root-page */ - int bBulk /* True if this call is part of a bulk load */ + int bBulk, /* True if this call is part of a bulk load */ + Pgno pgnoRoot /* Root page of b-tree being balanced */ ){ BtShared *pBt; /* The whole database */ int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ int nNew = 0; /* Number of pages in apNew[] */ int nOld; /* Number of pages in apOld[] */ @@ -8224,10 +8701,11 @@ } if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } + setMempageRoot(apOld[i], pgnoRoot); if( apOld[i]->nFree<0 ){ rc = btreeComputeFreeSpace(apOld[i]); if( rc ){ memset(apOld, 0, (i)*sizeof(MemPage*)); goto balance_cleanup; @@ -8573,11 +9051,11 @@ apNew[i] = pNew; nNew++; cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; } } @@ -8666,11 +9144,11 @@ ** If the sibling pages are not leaves, then the pointer map entry ** associated with the right-child of each sibling may also need to be ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ MemPage *pOld; MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; int iNew = 0; int iOld = 0; @@ -8861,11 +9339,11 @@ - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ + }else if( REQUIRE_PTRMAP && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ for(i=0; iaData[8]); @@ -8944,11 +9422,11 @@ */ rc = sqlite3PagerWrite(pRoot->pDbPage); if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM(pBt) ){ + if( REQUIRE_PTRMAP ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } if( rc ){ *ppChild = 0; @@ -9107,11 +9585,11 @@ ** copied either into the body of a database page or into the new ** pSpace buffer passed to the latter call to balance_nonroot(). */ u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, - pCur->hints&BTREE_BULKLOAD); + pCur->hints&BTREE_BULKLOAD, pCur->pgnoRoot); if( pFree ){ /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are ** now stored either on real database pages or within the ** new pSpace buffer, so it may be safely freed here. */ @@ -9215,10 +9693,11 @@ pBt = pPage->pBt; ovflPageSize = pBt->usableSize - 4; do{ rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); if( rc ) return rc; + setMempageRoot(pPage, pCur->pgnoRoot); if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ rc = SQLITE_CORRUPT_PAGE(pPage); }else{ if( iOffset+ovflPageSize<(u32)nTotal ){ ovflPgno = get4byte(pPage->aData); @@ -9488,10 +9967,11 @@ assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; pCur->info.nSize = 0; if( loc==0 ){ CellInfo info; + BtShared *pBt = p->pBt; assert( idx>=0 ); if( idx>=pPage->nCell ){ return SQLITE_CORRUPT_PAGE(pPage); } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -9504,11 +9984,11 @@ } BTREE_CLEAR_CELL(rc, pPage, oldCell, info); testcase( pCur->curFlags & BTCF_ValidOvfl ); invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload - && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) + && (!REQUIRE_PTRMAP || szNewminLocal) ){ /* Overwrite the old cell with the new if they are the same size. ** We could also try to do this if the old cell is smaller, then add ** the leftover space to the free list. But experiments show that ** doing that is no faster then skipping this optimization and just @@ -10093,11 +10573,12 @@ */ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ Pgno pgno, /* Page number to clear */ int freePageFlag, /* Deallocate page if true */ - i64 *pnChange /* Add number of Cells freed to this counter */ + i64 *pnChange, /* Add number of Cells freed to this counter */ + Pgno pgnoRoot ){ MemPage *pPage; int rc; unsigned char *pCell; int i; @@ -10108,10 +10589,11 @@ if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_PGNO(pgno); } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; + setMempageRoot(pPage, pgnoRoot); if( (pBt->openFlags & BTREE_SINGLE)==0 && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) ){ rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; @@ -10118,18 +10600,20 @@ } hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange, pgnoRoot); if( rc ) goto cleardatabasepage_out; } BTREE_CLEAR_CELL(rc, pPage, pCell, info); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); + rc = clearDatabasePage( + pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange, pgnoRoot + ); if( rc ) goto cleardatabasepage_out; if( pPage->intKey ) pnChange = 0; } if( pnChange ){ testcase( !pPage->intKey ); @@ -10171,11 +10655,11 @@ ** is the root of a table b-tree - if it is not, the following call is ** a no-op). */ if( p->hasIncrblobCur ){ invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); } - rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange, (Pgno)iTable); } sqlite3BtreeLeave(p); return rc; } @@ -11127,14 +11611,16 @@ } sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; - /* Make sure every page in the file is referenced - */ if( !bPartial ){ - for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){ + /* Make sure every page in the file is referenced. Skip this if the + ** database is currently being written by a CONCURRENT transaction (it + ** may fail as pages that were part of the free-list when the transaction + ** was opened cannot be counted). */ + for(i=1; ISCONCURRENT==0 && i<=sCheck.nCkPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ checkAppendMsg(&sCheck, "Page %u: never used", i); } #else @@ -11430,10 +11916,106 @@ /* ** Return the size of the header added to each page by this module. */ int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +/* +** This function is called to ensure that all locks required to commit the +** current write-transaction to the database file are held. If the db is +** in rollback mode, this means the EXCLUSIVE lock on the database file. +** +** Or, if this is an CONCURRENT transaction on a wal-mode database, the WRITER +** lock on the wal file. In this case this function also checks that the +** CONCURRENT transaction can be safely committed (does not commit with any +** other transaction committed since it was opened). +** +** SQLITE_OK is returned if successful. SQLITE_BUSY if the required locks +** cannot be obtained due to a conflicting lock. If the locks cannot be +** obtained for an CONCURRENT transaction due to a conflict with an already +** committed transaction, SQLITE_BUSY_SNAPSHOT is returned. Otherwise, if +** some other error (OOM, IO, etc.) occurs, the relevant SQLite error code +** is returned. +*/ +int sqlite3BtreeExclusiveLock(Btree *p){ + int rc; + Pgno pgno = 0; + BtShared *pBt = p->pBt; + assert( p->inTrans==TRANS_WRITE && pBt->pPage1 ); + sqlite3BtreeEnter(p); + rc = sqlite3PagerExclusiveLock(pBt->pPager, + (p->db->eConcurrent==CONCURRENT_SCHEMA) ? 0 : pBt->pPage1->pDbPage, + &pgno + ); +#ifdef SQLITE_OMIT_CONCURRENT + assert( pgno==0 ); +#else + if( rc==SQLITE_BUSY_SNAPSHOT && pgno ){ + PgHdr *pPg = 0; + int rc2 = sqlite3PagerGet(pBt->pPager, pgno, &pPg, 0); + if( rc2==SQLITE_OK ){ + int bWrite = -1; + const char *zObj = 0; + const char *zTab = 0; + char zContent[17]; + + if( pPg ){ + Pgno pgnoRoot = 0; + HashElem *pE; + Schema *pSchema; + u8 *aData = (u8*)sqlite3PagerGetData(pPg); + int i; + for(i=0; i<8; i++){ + static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + zContent[i*2] = hexdigits[(aData[i] >> 4)]; + zContent[i*2+1] = hexdigits[(aData[i] & 0xF)]; + } + zContent[16] = '\0'; + + pgnoRoot = ((MemPage*)sqlite3PagerGetExtra(pPg))->pgnoRoot; + bWrite = sqlite3PagerIswriteable(pPg); + sqlite3PagerUnref(pPg); + + pSchema = sqlite3SchemaGet(p->db, p); + if( pSchema ){ + for(pE=sqliteHashFirst(&pSchema->tblHash); pE; pE=sqliteHashNext(pE)){ + Table *pTab = (Table *)sqliteHashData(pE); + if( pTab->tnum==(int)pgnoRoot ){ + zObj = pTab->zName; + zTab = 0; + }else{ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->tnum==(int)pgnoRoot ){ + zObj = pIdx->zName; + zTab = pTab->zName; + } + } + } + } + } + } + + sqlite3_log(SQLITE_OK, + "cannot commit CONCURRENT transaction " + "- conflict at page %d " + "(%s page; part of db %s %s%s%s; content=%s...)", + (int)pgno, + (bWrite==0?"read-only":(bWrite>0?"read/write":"unknown")), + (zTab ? "index" : "table"), + (zTab ? zTab : ""), (zTab ? "." : ""), (zObj ? zObj : "UNKNOWN"), + zContent + ); + } + } +#endif + sqlite3BtreeLeave(p); + return rc; +} /* ** If no transaction is active and the database is not a temp-db, clear ** the in-memory pager cache. */ Index: src/btree.h ================================================================== --- src/btree.h +++ src/btree.h @@ -355,10 +355,12 @@ sqlite3_uint64 sqlite3BtreeSeekCount(Btree*); #else # define sqlite3BtreeSeekCount(X) 0 #endif +int sqlite3BtreeExclusiveLock(Btree *pBt); + #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); #endif int sqlite3BtreeCursorIsValidNN(BtCursor*); @@ -414,8 +416,7 @@ # define sqlite3BtreeHoldsMutex(X) 1 # define sqlite3BtreeHoldsAllMutexes(X) 1 # define sqlite3SchemaMutexHeld(X,Y,Z) 1 #endif - #endif /* SQLITE_BTREE_H */ Index: src/btreeInt.h ================================================================== --- src/btreeInt.h +++ src/btreeInt.h @@ -230,10 +230,11 @@ /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; typedef struct CellInfo CellInfo; +typedef struct BtreePtrmap BtreePtrmap; /* ** This is a magic string that appears at the beginning of every ** SQLite database in order to identify the file as a real database. ** @@ -273,10 +274,13 @@ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ Pgno pgno; /* Page number for this page */ +#ifndef SQLITE_OMIT_CONCURRENT + Pgno pgnoRoot; /* Root page of b-tree that this page belongs to */ +#endif /* Only the first 8 bytes (above) are zeroed by pager.c when a new page ** is allocated. All fields that follow must be initialized before use */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ @@ -454,10 +458,13 @@ BtShared *pNext; /* Next on a list of sharable BtShared structs */ BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ +#ifndef SQLITE_OMIT_CONCURRENT + BtreePtrmap *pMap; +#endif int nPreformatSize; /* Size of last cell written by TransferRow() */ }; /* ** Allowed values for BtShared.btsFlags @@ -671,16 +678,23 @@ ** if the database supports auto-vacuum or not. Because it is used ** within an expression that is an argument to another macro ** (sqliteMallocRaw), it is not possible to use conditional compilation. ** So, this macro is defined instead. */ -#ifndef SQLITE_OMIT_AUTOVACUUM +#ifdef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM(pBt) 0 +#else #define ISAUTOVACUUM(pBt) (pBt->autoVacuum) +#endif + +#ifdef SQLITE_OMIT_CONCURRENT +# define ISCONCURRENT 0 #else -#define ISAUTOVACUUM(pBt) 0 +# define ISCONCURRENT (pBt->pMap!=0) #endif +#define REQUIRE_PTRMAP (ISAUTOVACUUM(pBt) || ISCONCURRENT) /* ** This structure is passed around through all the PRAGMA integrity_check ** checking routines in order to keep track of some global state information. ** Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -87,12 +87,14 @@ int iDb, /* Index of the database containing the table to lock */ Pgno iTab, /* Root page number of the table to be locked */ u8 isWriteLock, /* True for a write lock */ const char *zName /* Name of the table to be locked */ ){ +#ifdef SQLITE_OMIT_CONCURRENT if( iDb==1 ) return; if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; +#endif lockTable(pParse, iDb, iTab, isWriteLock, zName); } /* ** Code an OP_TableLock instruction for each table locked by the @@ -5142,11 +5144,11 @@ if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ return; } v = sqlite3GetVdbe(pParse); if( !v ) return; - if( type!=TK_DEFERRED ){ + if( type==TK_IMMEDIATE || type==TK_EXCLUSIVE ){ for(i=0; inDb; i++){ int eTxnType; Btree *pBt = db->aDb[i].pBt; if( pBt && sqlite3BtreeIsReadonly(pBt) ){ eTxnType = 0; /* Read txn */ @@ -5157,11 +5159,11 @@ } sqlite3VdbeAddOp2(v, OP_Transaction, i, eTxnType); sqlite3VdbeUsesBtree(v, i); } } - sqlite3VdbeAddOp0(v, OP_AutoCommit); + sqlite3VdbeAddOp3(v, OP_AutoCommit, 0, 0, (type==TK_CONCURRENT)); } /* ** Generate VDBE code for a COMMIT or ROLLBACK statement. ** Code for ROLLBACK is generated if eType==TK_ROLLBACK. Otherwise Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -557,12 +557,13 @@ sqlite3_context *context, int NotUsed, sqlite3_value **NotUsed2 ){ sqlite_int64 r; + sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_randomness(sizeof(r), &r); + sqlite3FastRandomness(&db->sPrng, sizeof(r), &r); if( r<0 ){ /* We need to prevent a random number of 0x8000000000000000 ** (or -9223372036854775808) since when you do abs() of that ** number of you get the same value back again. To do this ** in a way that is testable, mask the sign bit off of negative @@ -584,19 +585,20 @@ int argc, sqlite3_value **argv ){ sqlite3_int64 n; unsigned char *p; + sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int64(argv[0]); if( n<1 ){ n = 1; } p = contextMalloc(context, n); if( p ){ - sqlite3_randomness(n, p); + sqlite3FastRandomness(&db->sPrng, n, p); sqlite3_result_blob(context, (char*)p, n, sqlite3_free); } } /* Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -3334,11 +3334,11 @@ db->nDb = 2; db->eOpenState = SQLITE_STATE_BUSY; db->aDb = db->aDbStatic; db->lookaside.bDisable = 1; db->lookaside.sz = 0; - + sqlite3FastPrngInit(&db->sPrng); assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; @@ -5054,10 +5054,39 @@ */ void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ sqlite3_free(pSnapshot); } #endif /* SQLITE_ENABLE_SNAPSHOT */ + +SQLITE_EXPERIMENTAL int sqlite3_wal_info( + sqlite3 *db, const char *zDb, + unsigned int *pnPrior, unsigned int *pnFrame +){ + int rc = SQLITE_OK; + +#ifndef SQLITE_OMIT_WAL + Btree *pBt; + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb<0 ){ + return SQLITE_ERROR; + } + pBt = db->aDb[iDb].pBt; + rc = sqlite3PagerWalInfo(sqlite3BtreePager(pBt), pnPrior, pnFrame); + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + + return rc; +} #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* ** Given the name of a compile-time option, return true if that option ** was used and false if not. Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -1222,10 +1222,14 @@ #endif #if OS_VXWORKS sem_t *pSem; /* Named POSIX semaphore */ char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ #endif +#ifdef SQLITE_SHARED_MAPPING + sqlite3_int64 nSharedMapping; /* Size of mapped region in bytes */ + void *pSharedMapping; /* Memory mapped region */ +#endif }; /* ** A lists of all unixInodeInfo objects. ** @@ -1378,10 +1382,17 @@ assert( unixFileMutexNotheld(pFile) ); if( ALWAYS(pInode) ){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); +#ifdef SQLITE_SHARED_MAPPING + if( pInode->pSharedMapping ){ + osMunmap(pInode->pSharedMapping, pInode->nSharedMapping); + pInode->pSharedMapping = 0; + pInode->nSharedMapping = 0; + } +#endif sqlite3_mutex_enter(pInode->pLockMutex); closePendingFds(pFile); sqlite3_mutex_leave(pInode->pLockMutex); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); @@ -2241,10 +2252,18 @@ /* ** Close the file. */ static int nolockClose(sqlite3_file *id) { +#ifdef SQLITE_SHARED_MAPPING + unixFile *pFd = (unixFile*)id; + if( pFd->pInode ){ + unixEnterMutex(); + releaseInodeInfo(pFd); + unixLeaveMutex(); + } +#endif return closeUnixFile(id); } /******************* End of the no-op lock implementation ********************* ******************************************************************************/ @@ -4085,10 +4104,13 @@ } *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; +#ifdef SQLITE_SHARED_MAPPING + if( pFile->pInode==0 ) +#endif if( pFile->mmapSize>0 ){ unixUnmapfile(pFile); rc = unixMapfile(pFile, -1); } } @@ -5283,10 +5305,13 @@ /* ** If it is currently memory mapped, unmap file pFd. */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); +#ifdef SQLITE_SHARED_MAPPING + if( pFd->pInode ) return; +#endif if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapSizeActual = 0; @@ -5413,10 +5438,32 @@ nMap = statbuf.st_size; } if( nMap>pFd->mmapSizeMax ){ nMap = pFd->mmapSizeMax; } + +#ifdef SQLITE_SHARED_MAPPING + if( pFd->pInode ){ + unixInodeInfo *pInode = pFd->pInode; + if( pFd->pMapRegion ) return SQLITE_OK; + unixEnterMutex(); + if( pInode->pSharedMapping==0 ){ + u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + unixLogError(SQLITE_OK, "mmap", pFd->zPath); + pFd->mmapSizeMax = 0; + }else{ + pInode->pSharedMapping = pNew; + pInode->nSharedMapping = nMap; + } + } + pFd->pMapRegion = pInode->pSharedMapping; + pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping; + unixLeaveMutex(); + return SQLITE_OK; + } +#endif assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ unixRemapfile(pFd, nMap); } @@ -5857,10 +5904,13 @@ if( pLockingStyle == &posixIoMethods #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE || pLockingStyle == &nfsIoMethods #endif +#ifdef SQLITE_SHARED_MAPPING + || pLockingStyle == &nolockIoMethods +#endif ){ unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ /* If an error occurred in findInodeInfo(), close the file descriptor Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -656,10 +656,13 @@ int errCode; /* One of several kinds of errors */ int nRec; /* Pages journalled since last j-header written */ u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ +#ifndef SQLITE_OMIT_CONCURRENT + Bitvec *pAllRead; /* Pages read within current CONCURRENT trans. */ +#endif sqlite3_file *fd; /* File descriptor for database */ sqlite3_file *jfd; /* File descriptor for main journal */ sqlite3_file *sjfd; /* File descriptor for sub-journal */ i64 journalOff; /* Current write offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ @@ -912,11 +915,13 @@ assert( p->eLock!=UNKNOWN_LOCK ); assert( pPager->errCode==SQLITE_OK ); if( !pagerUseWal(pPager) ){ assert( p->eLock>=RESERVED_LOCK ); } - assert( pPager->dbSize==pPager->dbOrigSize ); +#ifndef SQLITE_OMIT_CONCURRENT + assert( pPager->dbSize==pPager->dbOrigSize || pPager->pAllRead ); +#endif assert( pPager->dbOrigSize==pPager->dbFileSize ); assert( pPager->dbOrigSize==pPager->dbHintSize ); assert( pPager->setSuper==0 ); break; @@ -1819,10 +1824,57 @@ assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); } } return rc; } + +#ifndef SQLITE_OMIT_CONCURRENT +/* +** If they are not already, begin recording all pages read from the pager layer +** by the b-tree layer This is used by concurrent transactions. Return +** SQLITE_OK if successful, or an SQLite error code (SQLITE_NOMEM) if an error +** occurs. +*/ +int sqlite3PagerBeginConcurrent(Pager *pPager){ + int rc = SQLITE_OK; + if( pPager->pAllRead==0 ){ + pPager->pAllRead = sqlite3BitvecCreate(pPager->dbSize); + pPager->dbOrigSize = pPager->dbSize; + if( pPager->pAllRead==0 ){ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Stop recording all pages read from the pager layer by the b-tree layer +** and discard any current records. +*/ +void sqlite3PagerEndConcurrent(Pager *pPager){ + sqlite3BitvecDestroy(pPager->pAllRead); + pPager->pAllRead = 0; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Return true if the database is in wal mode. False otherwise. +*/ +int sqlite3PagerIsWal(Pager *pPager){ + return pPager->pWal!=0; +} +#endif /* SQLITE_OMIT_CONCURRENT */ + +/* +** Free the Pager.pInJournal and Pager.pAllRead bitvec objects. +*/ +static void pagerFreeBitvecs(Pager *pPager){ + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + sqlite3PagerEndConcurrent(pPager); +} /* ** This function is a no-op if the pager is in exclusive mode and not ** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN ** state. @@ -1844,12 +1896,11 @@ assert( pPager->eState==PAGER_READER || pPager->eState==PAGER_OPEN || pPager->eState==PAGER_ERROR ); - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; + pagerFreeBitvecs(pPager); releaseAllSavepoints(pPager); if( pagerUseWal(pPager) ){ assert( !isOpen(pPager->jfd) ); sqlite3WalEndReadTransaction(pPager->pWal); @@ -2112,12 +2163,11 @@ sqlite3PagerUnrefNotNull(p); } } #endif - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; + pagerFreeBitvecs(pPager); pPager->nRec = 0; if( rc==SQLITE_OK ){ if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ @@ -3143,12 +3193,28 @@ ** ** + Discard the cached page (if refcount==0), or ** + Reload page content from the database (if refcount>0). */ pPager->dbSize = pPager->dbOrigSize; - rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager, +#ifdef SQLITE_OMIT_CONCURRENT + 0 +#else + pPager->pAllRead!=0 +#endif + ); pList = sqlite3PcacheDirtyList(pPager->pPCache); + +#ifndef SQLITE_OMIT_CONCURRENT + /* If this is an CONCURRENT transaction, then page 1 must be reread from + ** the db file, even if it is not dirty. This is because the b-tree layer + ** may have already zeroed the nFree and iTrunk header fields. */ + if( rc==SQLITE_OK && (pList==0 || pList->pgno!=1) && pPager->pAllRead ){ + rc = pagerUndoCallback((void*)pPager, 1); + } +#endif + while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } @@ -3194,10 +3260,12 @@ nList = 0; for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ if( p->pgno<=nTruncate ){ ppNext = &p->pDirty; nList++; + PAGERTRACE(("TO-WAL %d page %d hash(%08x)\n", + PAGERID(pPager), p->pgno, pager_pagehash(p))); } } assert( pList ); }else{ nList = 1; @@ -4263,11 +4331,11 @@ || pPager->eState==PAGER_WRITER_DBMOD ); assert( assert_pager_state(pPager) ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerExclusiveLock(pPager); + rc = sqlite3PagerExclusiveLock(pPager, 0, 0); if( rc!=SQLITE_OK ) return rc; if( !pPager->noSync ){ assert( !pPager->tempFile ); if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ @@ -4614,10 +4682,16 @@ } pPager->aStat[PAGER_STAT_SPILL]++; pPg->pDirty = 0; if( pagerUseWal(pPager) ){ +#ifndef SQLITE_OMIT_CONCURRENT + /* If the transaction is a "BEGIN CONCURRENT" transaction, the page + ** cannot be flushed to disk. Return early in this case. */ + if( pPager->pAllRead ) return SQLITE_OK; +#endif + /* Write a single frame for this page to the log. */ rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } @@ -5445,10 +5519,27 @@ assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ pagerUnlockAndRollback(pPager); } } +#ifndef SQLITE_OMIT_CONCURRENT +/* +** If this pager is currently in a concurrent transaction (pAllRead!=0), +** then set the bit in the pAllRead vector to indicate that the transaction +** read from page pgno. Return SQLITE_OK if successful, or an SQLite error +** code (i.e. SQLITE_NOMEM) if an error occurs. +*/ +int sqlite3PagerUsePage(Pager *pPager, Pgno pgno){ + int rc = SQLITE_OK; + if( pPager->pAllRead && pgno<=pPager->dbOrigSize ){ + PAGERTRACE(("USING page %d\n", pgno)); + rc = sqlite3BitvecSet(pPager->pAllRead, pgno); + } + return rc; +} +#endif + /* ** The page getter methods each try to acquire a reference to a ** page with page number pgno. If the requested reference is ** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. ** @@ -5517,10 +5608,18 @@ assert( pPager->errCode==SQLITE_OK ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); assert( pPager->hasHeldSharedLock==1 ); + + /* If this is an CONCURRENT transaction and the page being read was + ** present in the database file when the transaction was opened, + ** mark it as read in the pAllRead vector. */ + if( (rc = sqlite3PagerUsePage(pPager, pgno))!=SQLITE_OK ){ + pPg = 0; + goto pager_acquire_err; + } if( pgno==0 ) return SQLITE_CORRUPT_BKPT; pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); if( pBase==0 ){ pPg = 0; @@ -5876,14 +5975,17 @@ /* ** Begin a write-transaction on the specified pager object. If a ** write-transaction has already been opened, this function is a no-op. ** -** If the exFlag argument is false, then acquire at least a RESERVED -** lock on the database file. If exFlag is true, then acquire at least +** If the exFlag argument is 0, then acquire at least a RESERVED +** lock on the database file. If exFlag is >0, then acquire at least ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. +** +** If (exFlag<0) and the database is in WAL mode, do not take any locks. +** The transaction will run in CONCURRENT mode instead. ** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when ** running in exclusive mode) or if the transaction does not require a @@ -5898,11 +6000,10 @@ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; if( pPager->eState==PAGER_READER ){ assert( pPager->pInJournal==0 ); - if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ @@ -5914,21 +6015,22 @@ } /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already - ** holds the write-lock. If possible, the upper layer will call it. - */ - rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + ** holds the write-lock. If possible, the upper layer will call it. */ + if( exFlag>=0 ){ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + } }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ rc = pagerLockDb(pPager, RESERVED_LOCK); - if( rc==SQLITE_OK && exFlag ){ + if( rc==SQLITE_OK && exFlag>0 ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } } if( rc==SQLITE_OK ){ @@ -6224,11 +6326,11 @@ /* ** Return TRUE if the page given in the argument was previously passed ** to sqlite3PagerWrite(). In other words, return TRUE if it is ok ** to change the content of the page. */ -#ifndef NDEBUG +#if !defined(SQLITE_OMIT_CONCURRENT) || !defined(NDEBUG) int sqlite3PagerIswriteable(DbPage *pPg){ return pPg->flags & PGHDR_WRITEABLE; } #endif @@ -6380,21 +6482,30 @@ } return rc; } /* -** This function may only be called while a write-transaction is active in -** rollback. If the connection is in WAL mode, this call is a no-op. -** Otherwise, if the connection does not already have an EXCLUSIVE lock on -** the database file, an attempt is made to obtain one. +** This function is called to ensure that all locks required to commit the +** current write-transaction to the database file are held. If the db is +** in rollback mode, this means the EXCLUSIVE lock on the database file. +** +** Or, if this is a non-CONCURRENT transaction on a wal-mode database, this +** function is a no-op. +** +** If this is an CONCURRENT transaction on a wal-mode database, this function +** attempts to obtain the WRITER lock on the wal file and also checks to +** see that the transaction can be safely committed (does not commit with +** any other transaction committed since it was opened). ** -** If the EXCLUSIVE lock is already held or the attempt to obtain it is -** successful, or the connection is in WAL mode, SQLITE_OK is returned. -** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is -** returned. +** If the required locks are already held or successfully obtained and +** the transaction can be committed, SQLITE_OK is returned. If a required lock +** cannot be obtained, SQLITE_BUSY is returned. Or, if the current transaction +** is CONCURRENT and cannot be committed due to a conflict, SQLITE_BUSY_SNAPSHOT +** is returned. Otherwise, if some other error occurs (IO error, OOM etc.), +** and SQLite error code is returned. */ -int sqlite3PagerExclusiveLock(Pager *pPager){ +int sqlite3PagerExclusiveLock(Pager *pPager, PgHdr *pPage1, Pgno *piConflict){ int rc = pPager->errCode; assert( assert_pager_state(pPager) ); if( rc==SQLITE_OK ){ assert( pPager->eState==PAGER_WRITER_CACHEMOD || pPager->eState==PAGER_WRITER_DBMOD @@ -6402,13 +6513,76 @@ ); assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } +#ifndef SQLITE_OMIT_CONCURRENT + else{ + if( pPager->pAllRead ){ + /* This is an CONCURRENT transaction. Attempt to lock the wal database + ** here. If SQLITE_BUSY (but not SQLITE_BUSY_SNAPSHOT) is returned, + ** invoke the busy-handler and try again for as long as it returns + ** non-zero. */ + do { + rc = sqlite3WalLockForCommit( + pPager->pWal, pPage1, pPager->pAllRead, piConflict + ); + }while( rc==SQLITE_BUSY + && pPager->xBusyHandler(pPager->pBusyHandlerArg) + ); + } + } +#endif /* SQLITE_OMIT_CONCURRENT */ + } + return rc; +} + +#ifndef SQLITE_OMIT_CONCURRENT +/* +** This function is called as part of committing an CONCURRENT transaction. +** At this point the wal WRITER lock is held, and all pages in the cache +** except for page 1 are compatible with the snapshot at the head of the +** wal file. +** +** This function updates the in-memory data structures and reloads the +** contents of page 1 so that the client is operating on the snapshot +** at the head of the wal file. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage *pPage1){ + int rc; + + assert( pPager->pWal && pPager->pAllRead ); + rc = sqlite3WalUpgradeSnapshot(pPager->pWal); + if( rc==SQLITE_OK ){ + rc = readDbPage(pPage1); } + return rc; } + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** Set the in-memory cache of the database file size to nSz pages. +*/ +void sqlite3PagerSetDbsize(Pager *pPager, Pgno nSz){ + pPager->dbSize = nSz; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** If this is a WAL mode connection and the WRITER lock is currently held, +** relinquish it. +*/ +void sqlite3PagerDropExclusiveLock(Pager *pPager){ + if( pagerUseWal(pPager) ){ + sqlite3WalEndWriteTransaction(pPager->pWal); + } +} +#endif /* SQLITE_OMIT_CONCURRENT */ + /* ** Sync the database file for the pager pPager. zSuper points to the name ** of a super-journal file that should be written into the individual ** journal file. zSuper may be NULL, which is interpreted as no @@ -7773,10 +7947,15 @@ assert( pPager->pWal ); sqlite3WalSnapshotUnlock(pPager->pWal); } #endif /* SQLITE_ENABLE_SNAPSHOT */ + +int sqlite3PagerWalInfo(Pager *pPager, u32 *pnPrior, u32 *pnFrame){ + return sqlite3WalInfo(pPager->pWal, pnPrior, pnFrame); +} + #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS /* ** A read-lock must be held on the pager when this function is called. If Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -161,17 +161,18 @@ /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int); -int sqlite3PagerExclusiveLock(Pager*); +int sqlite3PagerExclusiveLock(Pager*, DbPage *pPage1, Pgno*); int sqlite3PagerSync(Pager *pPager, const char *zSuper); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); + #ifndef SQLITE_OMIT_WAL int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); int sqlite3PagerWalSupported(Pager *pPager); int sqlite3PagerWalCallback(Pager *pPager); @@ -223,14 +224,32 @@ /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); void sqlite3PagerRekey(DbPage*, Pgno, u16); +#ifndef SQLITE_OMIT_CONCURRENT +int sqlite3PagerUsePage(Pager*, Pgno); +void sqlite3PagerEndConcurrent(Pager*); +int sqlite3PagerBeginConcurrent(Pager*); +void sqlite3PagerDropExclusiveLock(Pager*); +int sqlite3PagerUpgradeSnapshot(Pager *pPager, DbPage*); +void sqlite3PagerSetDbsize(Pager *pPager, Pgno); +int sqlite3PagerIsWal(Pager*); +#else +# define sqlite3PagerEndConcurrent(x) +# define sqlite3PagerUsePage(x, y) SQLITE_OK +#endif + +#if defined(SQLITE_DEBUG) || !defined(SQLITE_OMIT_CONCURRENT) +int sqlite3PagerIswriteable(DbPage*); +#endif + +int sqlite3PagerWalInfo(Pager*, u32 *pnPrior, u32 *pnFrame); + /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) Pgno sqlite3PagerPagenumber(DbPage*); - int sqlite3PagerIswriteable(DbPage*); #endif #ifdef SQLITE_TEST int *sqlite3PagerStats(Pager*); void sqlite3PagerRefdump(Pager*); void disable_simulated_io_errors(void); Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -106,10 +106,17 @@ ** UPDATE ON (a,b,c) ** ** Then the "b" IdList records the list "a,b,c". */ struct TrigEvent { int a; IdList * b; }; + +/* +** Generate a syntax error +*/ +static void parserSyntaxError(Parse *pParse, Token *p){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", p); +} struct FrameBound { int eType; Expr *pExpr; }; /* ** Disable lookaside memory allocation for objects that might be @@ -166,11 +173,20 @@ trans_opt ::= TRANSACTION nm. %type transtype {int} transtype(A) ::= . {A = TK_DEFERRED;} transtype(A) ::= DEFERRED(X). {A = @X; /*A-overwrites-X*/} transtype(A) ::= IMMEDIATE(X). {A = @X; /*A-overwrites-X*/} -transtype(A) ::= EXCLUSIVE(X). {A = @X; /*A-overwrites-X*/} +transtype(A) ::= ID(X). { + Token *p = &X; + if( p->n==9 && sqlite3_strnicmp(p->z,"exclusive",9)==0 ){ + A = TK_EXCLUSIVE; + }else if( p->n==10 && sqlite3_strnicmp(p->z,"concurrent",10)==0 ){ + A = TK_CONCURRENT; /*A-overwrites-X*/ + }else{ + parserSyntaxError(pParse, p); + } +} cmd ::= COMMIT|END(X) trans_opt. {sqlite3EndTransaction(pParse,@X);} cmd ::= ROLLBACK(X) trans_opt. {sqlite3EndTransaction(pParse,@X);} savepoint_opt ::= SAVEPOINT. savepoint_opt ::= . @@ -297,11 +313,10 @@ // An IDENTIFIER can be a generic identifier, or one of several // keywords. Any non-standard keyword can also be an identifier. // %token_class id ID|INDEXED. - // And "ids" is an identifer-or-string. // %token_class ids ID|STRING. // An identifier or a join-keyword @@ -1136,11 +1151,11 @@ ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ Token t = X; /*A-overwrites-X*/ assert( t.n>=2 ); if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + parserSyntaxError(pParse, &t); A = 0; }else{ A = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); if( A ) sqlite3GetInt32(&t.z[1], &A->iTable); } @@ -1931,10 +1946,11 @@ FUNCTION /* A function invocation */ UMINUS /* Unary minus */ UPLUS /* Unary plus */ TRUTH /* IS TRUE or IS FALSE or IS NOT TRUE or IS NOT FALSE */ REGISTER /* Reference to a VDBE register */ + CONCURRENT /* BEGIN CONCURRENT */ VECTOR /* Vector */ SELECT_COLUMN /* Choose a single column from a multi-column SELECT */ IF_NULL_ROW /* the if-null-row operator */ ASTERISK /* The "*" in count(*) and similar */ SPAN /* The span operator */ Index: src/pragma.h ================================================================== --- src/pragma.h +++ src/pragma.h @@ -439,10 +439,19 @@ /* ePragFlg: */ PragFlg_Result0, /* ColNames: */ 9, 1, /* iArg: */ 0 }, #endif #endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_ENABLE_NOOP_UPDATE) + {/* zName: */ "noop_update", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NoopUpdate }, +#endif #endif {/* zName: */ "optimize", /* ePragTyp: */ PragTyp_OPTIMIZE, /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, /* ColNames: */ 0, 0, @@ -655,6 +664,6 @@ /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 68 on by default, 78 total. */ +/* Number of pragmas: 68 on by default, 79 total. */ Index: src/random.c ================================================================== --- src/random.c +++ src/random.c @@ -126,10 +126,37 @@ chacha_block((u32*)wsdPrng.out, wsdPrng.s); wsdPrng.n = 64; } sqlite3_mutex_leave(mutex); } + +/* +** Initialize a fast PRNG. A Fast PRNG is called "fast" because it does +** not need a mutex to operate, though it does use a mutex to initialize. +** The quality of the randomness is not as good as the global PRNG. +*/ +void sqlite3FastPrngInit(FastPrng *pPrng){ + sqlite3_randomness(sizeof(*pPrng), pPrng); + pPrng->x |= 1; +} + +/* +** Generate N bytes of pseudo-randomness using a FastPrng +*/ +void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){ + unsigned char *pOut = (unsigned char*)P; + while( N-->0 ){ + /* "x" is a variant of LFSR called "Xorshift" by George Marsaglia */ + pPrng->x ^= pPrng->x <<13; + pPrng->x ^= pPrng->x >>7; + pPrng->x ^= pPrng->x <<17; + /* "y" is a LCG using Don Kunth's constants from MMIX */ + pPrng->y = (pPrng->y)*6364136223846793005LL + 1442695040888963407LL; + /* XOR the two streams together to give the final result */ + *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff; + } +} #ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -2270,11 +2270,11 @@ if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); sqlite3ProgressCheck(pParse); if( cnt>3 ){ - sqlite3_randomness(sizeof(cnt), &cnt); + sqlite3FastRandomness(&db->sPrng, sizeof(cnt), &cnt); } } pCol->zCnName = zName; pCol->hName = sqlite3StrIHash(zName); if( pX->fg.bNoExpand ){ Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -10654,10 +10654,35 @@ ** ** This interface is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Wal related information regarding the most recent COMMIT +** EXPERIMENTAL +** +** This function reports on the state of the wal file (if any) for database +** zDb, which should be "main", "temp", or the name of the attached database. +** Its results - the values written to the output parameters - are only +** defined if the most recent SQL command on the connection was a successful +** COMMIT that wrote data to wal-mode database zDb. +** +** Assuming the above conditions are met, output parameter (*pnFrame) is set +** to the total number of frames in the wal file. Parameter (*pnPrior) is +** set to the number of frames that were present in the wal file before the +** most recent transaction was committed. So that the number of frames written +** by the most recent transaction is (*pnFrame)-(*pnPrior). +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. It +** is not an error if this function is called at a time when the results +** are undefined. +*/ +SQLITE_EXPERIMENTAL int sqlite3_wal_info( + sqlite3 *db, const char *zDb, + unsigned int *pnPrior, unsigned int *pnFrame +); /* ** CAPI3REF: Serialize a database ** ** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1302,10 +1302,11 @@ typedef struct DbClientData DbClientData; typedef struct DbFixer DbFixer; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; +typedef struct FastPrng FastPrng; typedef struct FKey FKey; typedef struct FpDecode FpDecode; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; @@ -1428,10 +1429,18 @@ # define SQLITE_DEFAULT_SYNCHRONOUS 2 #endif #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS #endif + +/* +** State of a simple PRNG used for the per-connection and per-pager +** pseudo-random number generators. +*/ +struct FastPrng { + sqlite3_uint64 x, y; +}; /* ** Each database file to be accessed by the system is an instance ** of the following structure. There are normally two of these structures ** in the sqlite.aDb[] array. aDb[0] is the main database file and @@ -1680,10 +1689,11 @@ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ u32 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ + u8 eConcurrent; /* CONCURRENT_* value */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ u8 bBenignMalloc; /* Do not require OOMs if true */ u8 dfltLockMode; /* Default locking-mode for attached dbs */ signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ @@ -1693,10 +1703,11 @@ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 noSharedCache; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ u8 eOpenState; /* Current condition of the connection */ int nextPagesize; /* Pagesize after VACUUM if >0 */ + FastPrng sPrng; /* State of the per-connection PRNG */ i64 nChange; /* Value returned by sqlite3_changes() */ i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ @@ -1803,10 +1814,17 @@ #ifdef SQLITE_USER_AUTHENTICATION sqlite3_userauth auth; /* User authentication information */ #endif }; +/* +** Candidate values for sqlite3.eConcurrent +*/ +#define CONCURRENT_NONE 0 +#define CONCURRENT_OPEN 1 +#define CONCURRENT_SCHEMA 2 + /* ** A macro to discover the encoding of a database. */ #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) #define ENC(db) ((db)->enc) @@ -1865,10 +1883,13 @@ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ #define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ #define SQLITE_FkNoAction HI(0x00008) /* Treat all FK as NO ACTION */ +/* Flags used by the Pragma noop_update enhancement */ +#define SQLITE_NoopUpdate HI(0x0001000) /* UPDATE operations are no-ops */ + /* Flags used only if debugging */ #ifdef SQLITE_DEBUG #define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ #define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ #define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ @@ -5066,10 +5087,12 @@ Vdbe *sqlite3GetVdbe(Parse*); #ifndef SQLITE_UNTESTABLE void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); #endif +void sqlite3FastPrngInit(FastPrng*); +void sqlite3FastRandomness(FastPrng*, int N, void *P); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3EndTransaction(Parse*,int); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -8683,10 +8683,45 @@ rc = sqlite3_db_config(db, SQLITE_DBCONFIG_MAINDBNAME, "icecube"); Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_OK; } } + +/* +** Usage: sqlite3_wal_info DB DBNAME +*/ +static int SQLITE_TCLAPI test_wal_info( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + unsigned int nPrior; + unsigned int nFrame; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + rc = sqlite3_wal_info(db, zName, &nPrior, &nFrame); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else{ + Tcl_Obj *pNew = Tcl_NewObj(); + Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nPrior)); + Tcl_ListObjAppendElement(interp, pNew, Tcl_NewWideIntObj((i64)nFrame)); + Tcl_SetObjResult(interp, pNew); + } + return TCL_OK; +} /* ** Usage: sqlite3_mmap_warm DB DBNAME */ static int SQLITE_TCLAPI test_mmap_warm( @@ -9286,12 +9321,13 @@ { "sqlite3_snapshot_recover", test_snapshot_recover, 0 }, { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 }, { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 }, { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 }, #endif - { "sqlite3_delete_database", test_delete_database, 0 }, - { "atomic_batch_write", test_atomic_batch_write, 0 }, + { "sqlite3_delete_database", test_delete_database, 0 }, + { "sqlite3_wal_info", test_wal_info, 0 }, + { "atomic_batch_write", test_atomic_batch_write, 0 }, { "sqlite3_mmap_warm", test_mmap_warm, 0 }, { "sqlite3_config_sorterref", test_config_sorterref, 0 }, { "sqlite3_autovacuum_pages", test_autovacuum_pages, 0 }, { "decode_hexdb", test_decode_hexdb, 0 }, { "test_write_db", test_write_db, 0 }, Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -674,10 +674,16 @@ #ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "truncate_opt", "1", TCL_GLOBAL_ONLY); #endif + +#ifndef SQLITE_OMIT_CONCURRENT + Tcl_SetVar2(interp, "sqlite_options", "concurrent", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "concurrent", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_UTF16 Tcl_SetVar2(interp, "sqlite_options", "utf16", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "utf16", "1", TCL_GLOBAL_ONLY); Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -463,10 +463,21 @@ ** column to be updated, make sure we have authorization to change ** that column. */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ +#if defined(SQLITE_ENABLE_NOOP_UPDATE) && !defined(SQLITE_OMIT_FLAG_PRAGMAS) + if( db->flags & SQLITE_NoopUpdate ){ + Token x; + sqlite3ExprDelete(db, pChanges->a[i].pExpr); + x.z = pChanges->a[i].zEName; + x.n = sqlite3Strlen30(x.z); + pChanges->a[i].pExpr = + sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprAlloc(db, TK_ID, &x, 0), 0); + if( db->mallocFailed ) goto update_cleanup; + } +#endif u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName); /* If this is an UPDATE with a FROM clause, do not resolve expressions ** here. The call to sqlite3Select() below will do that. */ if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; Index: src/vacuum.c ================================================================== --- src/vacuum.c +++ src/vacuum.c @@ -394,10 +394,11 @@ ** by manually setting the autoCommit flag to true and detaching the ** vacuum database. The vacuum_db journal file is deleted when the pager ** is closed by the DETACH. */ db->autoCommit = 1; + assert( db->eConcurrent==0 ); if( pDb ){ sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3823,10 +3823,11 @@ /* Determine whether or not this is a transaction savepoint. If so, ** and this is a RELEASE command, then the current transaction ** is committed. */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; + assert( db->eConcurrent==0 || db->isTransactionSavepoint==0 ); if( isTransaction && p1==SAVEPOINT_RELEASE ){ if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; } db->autoCommit = 1; @@ -3909,54 +3910,73 @@ goto vdbe_return; } break; } -/* Opcode: AutoCommit P1 P2 * * * +/* Opcode: AutoCommit P1 P2 P3 * * ** ** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll ** back any currently active btree transactions. If there are any active ** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if ** there are active writing VMs or active VMs that use shared cache. +** +** If P3 is non-zero, then this instruction is being executed as part of +** a "BEGIN CONCURRENT" command. ** ** This instruction causes the VM to halt. */ case OP_AutoCommit: { int desiredAutoCommit; int iRollback; + int bConcurrent; + int hrc; desiredAutoCommit = pOp->p1; iRollback = pOp->p2; + bConcurrent = pOp->p3; assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); assert( desiredAutoCommit==1 || iRollback==0 ); + assert( desiredAutoCommit==0 || bConcurrent==0 ); + assert( db->autoCommit==0 || db->eConcurrent==CONCURRENT_NONE ); assert( db->nVdbeActive>0 ); /* At least this one VM is active */ assert( p->bIsReader ); if( desiredAutoCommit!=db->autoCommit ){ if( iRollback ){ assert( desiredAutoCommit==1 ); sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); db->autoCommit = 1; - }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ - /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. - */ + db->eConcurrent = CONCURRENT_NONE; + }else if( desiredAutoCommit + && (db->nVdbeWrite>0 || (db->eConcurrent && db->nVdbeActive>1)) ){ + /* A transaction may only be committed if there are no other active + ** writer VMs. If the transaction is CONCURRENT, then it may only be + ** committed if there are no active VMs at all (readers or writers). + ** + ** If this instruction is a COMMIT and the transaction may not be + ** committed due to one of the conditions above, return an error + ** indicating that other VMs must complete before the COMMIT can + ** be processed. */ sqlite3VdbeError(p, "cannot commit transaction - " "SQL statements in progress"); rc = SQLITE_BUSY; goto abort_due_to_error; }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ goto vdbe_return; }else{ db->autoCommit = (u8)desiredAutoCommit; } - if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + hrc = sqlite3VdbeHalt(p); + if( (hrc & 0xFF)==SQLITE_BUSY ){ p->pc = (int)(pOp - aOp); db->autoCommit = (u8)(1-desiredAutoCommit); - p->rc = rc = SQLITE_BUSY; + p->rc = hrc; + rc = SQLITE_BUSY; goto vdbe_return; } + assert( bConcurrent==CONCURRENT_NONE || bConcurrent==CONCURRENT_OPEN ); + db->eConcurrent = (u8)bConcurrent; sqlite3CloseSavepoints(db); if( p->rc==SQLITE_OK ){ rc = SQLITE_DONE; }else{ rc = SQLITE_ERROR; @@ -4165,10 +4185,21 @@ assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); +#ifndef SQLITE_OMIT_CONCURRENT + if( db->eConcurrent + && (pOp->p2==BTREE_USER_VERSION || pOp->p2==BTREE_APPLICATION_ID) + ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, "cannot modify %s within CONCURRENT transaction", + pOp->p2==BTREE_USER_VERSION ? "user_version" : "application_id" + ); + goto abort_due_to_error; + } +#endif /* See note about index shifting on OP_ReadCookie */ rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ *(u32*)&pDb->pSchema->schema_cookie = *(u32*)&pOp->p3 - pOp->p5; @@ -4314,10 +4345,15 @@ assert( DbMaskTest(p->btreeMask, iDb) ); pDb = &db->aDb[iDb]; pX = pDb->pBt; assert( pX!=0 ); if( pOp->opcode==OP_OpenWrite ){ +#ifndef SQLITE_OMIT_CONCURRENT + if( db->eConcurrent==CONCURRENT_OPEN && p2==1 && iDb!=1 ){ + db->eConcurrent = CONCURRENT_SCHEMA; + } +#endif assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; @@ -8066,10 +8102,15 @@ ** P4 contains a pointer to the name of the table being locked. This is only ** used to generate an error message if the lock cannot be obtained. */ case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; +#ifndef SQLITE_OMIT_CONCURRENT + if( isWriteLock && db->eConcurrent && pOp->p2==1 && pOp->p1!=1 ){ + db->eConcurrent = CONCURRENT_SCHEMA; + } +#endif if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){ int p1 = pOp->p1; assert( p1>=0 && p1nDb ); assert( DbMaskTest(p->btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -2941,14 +2941,31 @@ && sqlite3PagerIsMemdb(pPager)==0 ){ assert( i!=1 ); nTrans++; } - rc = sqlite3PagerExclusiveLock(pPager); + rc = sqlite3BtreeExclusiveLock(pBt); sqlite3BtreeLeave(pBt); } } + +#ifndef SQLITE_OMIT_CONCURRENT + if( db->eConcurrent && (rc & 0xFF)==SQLITE_BUSY ){ + /* An SQLITE_BUSY or SQLITE_BUSY_SNAPSHOT was encountered while + ** attempting to take the WRITER lock on a wal file. Release the + ** WRITER locks on all wal files and return early. */ + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + sqlite3BtreeEnter(pBt); + sqlite3PagerDropExclusiveLock(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + } + } + } +#endif + if( rc!=SQLITE_OK ){ return rc; } /* If there are any write-transactions at all, invoke the commit hook */ @@ -3346,10 +3363,11 @@ ** so, abort any other statements this handle currently has active. */ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->eConcurrent = CONCURRENT_NONE; p->nChange = 0; } } } @@ -3384,13 +3402,13 @@ ** or hit an 'OR FAIL' constraint and there are no deferred foreign ** key constraints to hold up the transaction. This means a commit ** is required. */ rc = vdbeCommit(db, p); } - if( rc==SQLITE_BUSY && p->readOnly ){ + if( (rc & 0xFF)==SQLITE_BUSY && p->readOnly ){ sqlite3VdbeLeave(p); - return SQLITE_BUSY; + return rc; }else if( rc!=SQLITE_OK ){ sqlite3SystemError(db, rc); p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -3414,10 +3432,11 @@ eStatementOp = SAVEPOINT_ROLLBACK; }else{ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->eConcurrent = CONCURRENT_NONE; p->nChange = 0; } } /* If eStatementOp is non-zero, then a statement transaction needs to @@ -3435,10 +3454,11 @@ p->zErrMsg = 0; } sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + db->eConcurrent = CONCURRENT_NONE; p->nChange = 0; } } /* If this was an INSERT, UPDATE or DELETE and no statement transaction Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -524,12 +524,14 @@ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ + u32 nPriorFrame; /* For sqlite3WalInfo() */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ + FastPrng sPrng; /* Random number generator */ #ifdef SQLITE_USE_SEH u32 lockMask; /* Mask of locks held */ void *pFree; /* Pointer to sqlite3_free() if exception thrown */ u32 *pWiValue; /* Value to write into apWiData[iWiPg] */ int iWiPg; /* Write pWiValue into apWiData[iWiPg] */ @@ -1062,11 +1064,11 @@ /* ** Set or release locks on the WAL. Locks are either shared or exclusive. ** A lock cannot be moved directly between shared and exclusive - it must go -** through the unlocked state first. +** through the concurrent state first. ** ** In locking_mode=EXCLUSIVE, all of these routines become no-ops. */ static int walLockShared(Wal *pWal, int lockIdx){ int rc; @@ -1380,11 +1382,11 @@ int iLock; /* Lock offset to lock for checkpoint */ /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte. - ** If successful, the same bytes that are locked here are unlocked before + ** If successful, the same bytes that are locked here are concurrent before ** this function returns. */ assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); @@ -1704,10 +1706,11 @@ pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + sqlite3FastPrngInit(&pRet->sPrng); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ @@ -2333,11 +2336,11 @@ SEH_INJECT_FAULT; if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; - sqlite3_randomness(4, &salt1); + sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as @@ -2538,10 +2541,53 @@ sqlite3_free((void *)pWal->apWiData); sqlite3_free(pWal); } return rc; } + +/* +** Try to copy the wal-index header from shared-memory into (*pHdr). Return +** zero if successful or non-zero otherwise. If the header is corrupted +** (either because the two copies are inconsistent or because the checksum +** values are incorrect), the read fails and non-zero is returned. +*/ +static int walIndexLoadHdr(Wal *pWal, WalIndexHdr *pHdr){ + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h2; /* Second copy of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ + + /* The first page of the wal-index must be mapped at this point. */ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Read the header. This might happen concurrently with a write to the + ** same area of shared memory on a different CPU in a SMP, + ** meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return non-zero. + ** + ** There are two copies of the header at the beginning of the wal-index. + ** When reading, read [0] first then [1]. Writes are in the reverse order. + ** Memory barriers are used to prevent the compiler or the hardware from + ** reordering the reads and writes. + */ + aHdr = walIndexHdr(pWal); + memcpy(pHdr, (void *)&aHdr[0], sizeof(h2)); + walShmBarrier(pWal); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); + + if( memcmp(&h2, pHdr, sizeof(h2))!=0 ){ + return 1; /* Dirty read */ + } + if( h2.isInit==0 ){ + return 1; /* Malformed header - probably all zeros */ + } + walChecksumBytes(1, (u8*)&h2, sizeof(h2)-sizeof(h2.aCksum), 0, aCksum); + if( aCksum[0]!=h2.aCksum[0] || aCksum[1]!=h2.aCksum[1] ){ + return 1; /* Checksum does not match */ + } + + return 0; +} /* ** Try to read the wal-index header. Return 0 on success and 1 if ** there is a problem. ** @@ -2557,47 +2603,14 @@ ** ** If the checksum cannot be verified return non-zero. If the header ** is read successfully and the checksum verified, return zero. */ static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ - u32 aCksum[2]; /* Checksum on the header content */ - WalIndexHdr h1, h2; /* Two copies of the header content */ - WalIndexHdr volatile *aHdr; /* Header in shared memory */ - - /* The first page of the wal-index must be mapped at this point. */ - assert( pWal->nWiData>0 && pWal->apWiData[0] ); - - /* Read the header. This might happen concurrently with a write to the - ** same area of shared memory on a different CPU in a SMP, - ** meaning it is possible that an inconsistent snapshot is read - ** from the file. If this happens, return non-zero. - ** - ** tag-20200519-1: - ** There are two copies of the header at the beginning of the wal-index. - ** When reading, read [0] first then [1]. Writes are in the reverse order. - ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. TSAN and similar tools can sometimes - ** give false-positive warnings about these accesses because the tools do not - ** account for the double-read and the memory barrier. The use of mutexes - ** here would be problematic as the memory being accessed is potentially - ** shared among multiple processes and not all mutex implementations work - ** reliably in that environment. - */ - aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ - walShmBarrier(pWal); - memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); - - if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ - return 1; /* Dirty read */ - } - if( h1.isInit==0 ){ - return 1; /* Malformed header - probably all zeros */ - } - walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); - if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ - return 1; /* Checksum does not match */ + WalIndexHdr h1; /* Copy of the header content */ + + if( walIndexLoadHdr(pWal, &h1) ){ + return 1; } if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ *pChanged = 1; memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); @@ -3377,10 +3390,11 @@ testcase( (rc&0xff)==SQLITE_BUSY ); testcase( (rc&0xff)==SQLITE_IOERR ); testcase( rc==SQLITE_PROTOCOL ); testcase( rc==SQLITE_OK ); + pWal->nPriorFrame = pWal->hdr.mxFrame; #ifdef SQLITE_ENABLE_SNAPSHOT if( rc==SQLITE_OK ){ if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ /* At this point the client has a lock on an aReadMark[] slot holding ** a value equal to or smaller than pSnapshot->mxFrame, but pWal->hdr @@ -3504,12 +3518,11 @@ if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } - /* Search the hash table or tables for an entry matching page number - ** pgno. Each iteration of the following for() loop searches one + /* Each iteration of the following for() loop searches one ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). ** ** This code might run concurrently to the code in walIndexAppend() ** that adds entries to the wal-index (and possibly to this hash ** table). This means the value just read from the hash @@ -3637,10 +3650,39 @@ return pWal->hdr.nPage; } return 0; } +/* +** Take the WRITER lock on the WAL file. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. This routine does not invoke any +** busy-handler callbacks, that is done at a higher level. +*/ +static int walWriteLock(Wal *pWal){ + int rc; + + /* Cannot start a write transaction without first holding a read lock */ + assert( pWal->readLock>=0 ); + assert( pWal->writeLock==0 ); + assert( pWal->iReCksum==0 ); + + /* If this is a read-only connection, obtaining a write-lock is not + ** possible. In this case return SQLITE_READONLY. Otherwise, attempt + ** to grab the WRITER lock. Set Wal.writeLock to true and return + ** SQLITE_OK if successful, or leave Wal.writeLock clear and return + ** an SQLite error code (possibly SQLITE_BUSY) otherwise. */ + if( pWal->readOnly ){ + rc = SQLITE_READONLY; + }else{ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + } + } + + return rc; +} /* ** This function starts a write transaction on the WAL. ** ** A read transaction must have already been started by a prior call @@ -3663,46 +3705,241 @@ if( pWal->writeLock ){ assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); return SQLITE_OK; } #endif - - /* Cannot start a write transaction without first holding a read - ** transaction. */ - assert( pWal->readLock>=0 ); - assert( pWal->writeLock==0 && pWal->iReCksum==0 ); - - if( pWal->readOnly ){ - return SQLITE_READONLY; - } - - /* Only one writer allowed at a time. Get the write lock. Return - ** SQLITE_BUSY if unable. - */ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc ){ - return rc; - } - pWal->writeLock = 1; - - /* If another connection has written to the database file since the - ** time the read transaction on this connection was started, then - ** the write is disallowed. - */ - SEH_TRY { - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + + rc = walWriteLock(pWal); + if( rc==SQLITE_OK ){ + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. Release the WRITER lock and return + ** SQLITE_BUSY_SNAPSHOT in this case. */ + SEH_TRY { + if( memcmp(&pWal->hdr, (void*)walIndexHdr(pWal),sizeof(WalIndexHdr))!=0 ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + if( rc!=SQLITE_OK ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } + } + return rc; +} + +/* +** This function is called by a writer that has a read-lock on aReadmark[0] +** (pWal->readLock==0). This function relinquishes that lock and takes a +** lock on a different aReadmark[] slot. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walUpgradeReadlock(Wal *pWal){ + int cnt; + int rc; + assert( pWal->writeLock && pWal->readLock==0 ); + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + return rc; +} + + +#ifndef SQLITE_OMIT_CONCURRENT +/* +** This function does the work of sqlite3WalLockForCommit(). The difference +** between this function and sqlite3WalLockForCommit() is that the latter +** encloses everything in a SEH_TRY {} block. +*/ +static int walLockForCommit( + Wal *pWal, + PgHdr *pPg1, + Bitvec *pAllRead, + Pgno *piConflict +){ + int rc = walWriteLock(pWal); + + /* If the database has been modified since this transaction was started, + ** check if it is still possible to commit. The transaction can be + ** committed if: + ** + ** a) None of the pages in pList have been modified since the + ** transaction opened, and + ** + ** b) The database schema cookie has not been modified since the + ** transaction was started. + */ + if( rc==SQLITE_OK ){ + WalIndexHdr head; + + if( walIndexLoadHdr(pWal, &head) ){ + /* This branch is taken if the wal-index header is corrupted. This + ** occurs if some other writer has crashed while committing a + ** transaction to this database since the current concurrent transaction + ** was opened. */ rc = SQLITE_BUSY_SNAPSHOT; - } - } - SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) - - if( rc!=SQLITE_OK ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - } + }else if( memcmp(&pWal->hdr, (void*)&head, sizeof(WalIndexHdr))!=0 ){ + int iHash; + int iLast = walFramePage(head.mxFrame); + u32 iFirst = pWal->hdr.mxFrame+1; /* First wal frame to check */ + if( memcmp(pWal->hdr.aSalt, (u32*)head.aSalt, sizeof(u32)*2) ){ + assert( pWal->readLock==0 ); + iFirst = 1; + } + if( pPg1==0 ){ + /* If pPg1==0, then the current transaction modified the database + ** schema. This means it conflicts with all other transactions. */ + *piConflict = 1; + rc = SQLITE_BUSY_SNAPSHOT; + } + for(iHash=walFramePage(iFirst); rc==SQLITE_OK && iHash<=iLast; iHash++){ + WalHashLoc sLoc; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc==SQLITE_OK ){ + u32 i, iMin, iMax; + assert( head.mxFrame>=sLoc.iZero ); + iMin = (sLoc.iZero >= iFirst) ? 1 : (iFirst - sLoc.iZero); + iMax = (iHash==0) ? HASHTABLE_NPAGE_ONE : HASHTABLE_NPAGE; + if( iMax>(head.mxFrame-sLoc.iZero) ) iMax = (head.mxFrame-sLoc.iZero); + for(i=iMin; rc==SQLITE_OK && i<=iMax; i++){ + PgHdr *pPg; + if( sLoc.aPgno[i-1]==1 ){ + /* Check that the schema cookie has not been modified. If + ** it has not, the commit can proceed. */ + u8 aNew[4]; + u8 *aOld = &((u8*)pPg1->pData)[40]; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + iOffset = walFrameOffset(i+sLoc.iZero, sz) + WAL_FRAME_HDRSIZE+40; + rc = sqlite3OsRead(pWal->pWalFd, aNew, sizeof(aNew), iOffset); + if( rc==SQLITE_OK && memcmp(aOld, aNew, sizeof(aNew)) ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + }else if( sqlite3BitvecTestNotNull(pAllRead, sLoc.aPgno[i-1]) ){ + *piConflict = sLoc.aPgno[i-1]; + rc = SQLITE_BUSY_SNAPSHOT; + }else + if( (pPg = sqlite3PagerLookup(pPg1->pPager, sLoc.aPgno[i-1])) ){ + /* Page aPgno[i-1], which is present in the pager cache, has been + ** modified since the current CONCURRENT transaction was started. + ** However it was not read by the current transaction, so is not + ** a conflict. There are two possibilities: (a) the page was + ** allocated at the of the file by the current transaction or + ** (b) was present in the cache at the start of the transaction. + ** + ** For case (a), do nothing. This page will be moved within the + ** database file by the commit code to avoid the conflict. The + ** call to PagerUnref() is to release the reference grabbed by + ** the sqlite3PagerLookup() above. + ** + ** In case (b), drop the page from the cache - otherwise + ** following the snapshot upgrade the cache would be inconsistent + ** with the database as stored on disk. */ + if( sqlite3PagerIswriteable(pPg) ){ + sqlite3PagerUnref(pPg); + }else{ + sqlite3PcacheDrop(pPg); + } + } + } + } + } + } + } + + pWal->nPriorFrame = pWal->hdr.mxFrame; + return rc; +} + +/* +** This function is only ever called when committing a "BEGIN CONCURRENT" +** transaction. It may be assumed that no frames have been written to +** the wal file. The second parameter is a pointer to the in-memory +** representation of page 1 of the database (which may or may not be +** dirty). The third is a bitvec with a bit set for each page in the +** database file that was read by the current concurrent transaction. +** +** This function performs three tasks: +** +** 1) It obtains the WRITER lock on the wal file, +** +** 2) It checks that there are no conflicts between the current +** transaction and any transactions committed to the wal file since +** it was opened, and +** +** 3) It ejects any non-dirty pages from the page-cache that have been +** written by another client since the CONCURRENT transaction was started +** (so as to avoid ending up with an inconsistent cache after the +** current transaction is committed). +** +** If no error occurs and the caller may proceed with committing the +** transaction, SQLITE_OK is returned. SQLITE_BUSY is returned if the WRITER +** lock cannot be obtained. Or, if the WRITER lock can be obtained but there +** are conflicts with a committed transaction, SQLITE_BUSY_SNAPSHOT. Finally, +** if an error (i.e. an OOM condition or IO error), an SQLite error code +** is returned. +*/ +int sqlite3WalLockForCommit( + Wal *pWal, + PgHdr *pPg1, + Bitvec *pAllRead, + Pgno *piConflict +){ + int rc = SQLITE_OK; + SEH_TRY { + rc = walLockForCommit(pWal, pPg1, pAllRead, piConflict); + } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + return rc; +} + +/* !defined(SQLITE_OMIT_CONCURRENT) +** +** This function is called as part of committing an CONCURRENT transaction. +** It is assumed that sqlite3WalLockForCommit() has already been successfully +** called and so (a) the WRITER lock is held and (b) it is known that the +** wal-index-header stored in shared memory is not corrupt. +** +** Before returning, this function upgrades the client so that it is +** operating on the database snapshot currently at the head of the wal file +** (even if the CONCURRENT transaction ran against an older snapshot). +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +int sqlite3WalUpgradeSnapshot(Wal *pWal){ + int rc = SQLITE_OK; + assert( pWal->writeLock ); + + SEH_TRY { + assert( pWal->szPage==pWal->hdr.szPage ); + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + assert( pWal->szPage==pWal->hdr.szPage || pWal->szPage==0 ); + pWal->szPage = pWal->hdr.szPage; + + /* If this client has its read-lock on slot aReadmark[0] and the entire + ** wal has not been checkpointed, switch it to a different slot. Otherwise + ** any reads performed between now and committing the transaction will + ** read from the old snapshot - not the one just upgraded to. */ + if( pWal->readLock==0 && pWal->hdr.mxFrame!=walCkptInfo(pWal)->nBackfill ){ + rc = walUpgradeReadlock(pWal); + } + } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) return rc; } +#endif /* SQLITE_OMIT_CONCURRENT */ /* ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. */ @@ -3726,24 +3963,36 @@ ** returned to the caller. ** ** Otherwise, if the callback function does not return an error, this ** function returns SQLITE_OK. */ -int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ +int sqlite3WalUndo( + Wal *pWal, + int (*xUndo)(void *, Pgno), + void *pUndoCtx, + int bConcurrent /* True if this is a CONCURRENT transaction */ +){ int rc = SQLITE_OK; - if( ALWAYS(pWal->writeLock) ){ + if( pWal->writeLock ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. + */ SEH_TRY { - /* Restore the clients cache of the wal-index header to the state it - ** was in before the client began writing to the database. - */ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); +#ifndef SQLITE_OMIT_CONCURRENT + if( bConcurrent ){ + pWal->hdr.aCksum[0]++; + } +#else + UNUSED_PARAMETER(bConcurrent); +#endif - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number ** is passed as the second argument is (a) in the cache and ** (b) has an outstanding reference, then xUndo is either a no-op @@ -3757,12 +4006,11 @@ */ assert( walFramePgno(pWal, iFrame)!=1 ); rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); - } - SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + } SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; } /* @@ -3770,11 +4018,10 @@ ** values. This function populates the array with values required to ** "rollback" the write position of the WAL handle back to the current ** point in the event of a savepoint rollback (via WalSavepointUndo()). */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ - assert( pWal->writeLock ); aWalData[0] = pWal->hdr.mxFrame; aWalData[1] = pWal->hdr.aFrameCksum[0]; aWalData[2] = pWal->hdr.aFrameCksum[1]; aWalData[3] = pWal->nCkpt; } @@ -3786,11 +4033,11 @@ ** by a call to WalSavepoint(). */ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ int rc = SQLITE_OK; - assert( pWal->writeLock ); + assert( pWal->writeLock || aWalData[0]==pWal->hdr.mxFrame ); assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); if( aWalData[3]!=pWal->nCkpt ){ /* This savepoint was opened immediately after the write-transaction ** was started. Right after that, the writer decided to wrap around @@ -3825,18 +4072,17 @@ ** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned ** if an error occurs. */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; - int cnt; if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ u32 salt1; - sqlite3_randomness(4, &salt1); + sqlite3FastRandomness(&pWal->sPrng, 4, &salt1); rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ /* If all readers are using WAL_READ_LOCK(0) (in other words if no ** readers are currently using the WAL), then the transactions ** frames will overwrite the start of the existing log. Update the @@ -3846,25 +4092,25 @@ ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() ** to handle if this transaction is rolled back. */ walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + pWal->nPriorFrame = 0; }else if( rc!=SQLITE_BUSY ){ return rc; } } - walUnlockShared(pWal, WAL_READ_LOCK(0)); - pWal->readLock = -1; - cnt = 0; - do{ - int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); - }while( rc==WAL_RETRY ); - assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ - testcase( (rc&0xff)==SQLITE_IOERR ); - testcase( rc==SQLITE_PROTOCOL ); - testcase( rc==SQLITE_OK ); + + /* Regardless of whether or not the wal file was restarted, change the + ** read-lock held by this client to a slot other than aReadmark[0]. + ** Clients with a lock on aReadmark[0] read from the database file + ** only - never from the wal file. This means that if a writer holding + ** a lock on aReadmark[0] were to commit a transaction but not close the + ** read-transaction, subsequent read operations would read directly from + ** the database file - ignoring the new pages just appended + ** to the wal file. */ + rc = walUpgradeReadlock(pWal); } return rc; } /* @@ -4044,11 +4290,11 @@ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + if( pWal->nCkpt==0 ) sqlite3FastRandomness(&pWal->sPrng, 8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); @@ -4122,10 +4368,11 @@ if( rc ) return rc; pLast = p; iOffset += szFrame; p->flags |= PGHDR_WAL_APPEND; } + /* Recalculate checksums within the wal file if required. */ if( isCommit && pWal->iReCksum ){ rc = walRewriteChecksums(pWal, iFrame); if( rc ) return rc; @@ -4337,13 +4584,13 @@ /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } - + /* If no error occurred, set the output variables. */ if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; SEH_INJECT_FAULT; if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); @@ -4573,7 +4820,19 @@ /* Return the sqlite3_file object for the WAL file */ sqlite3_file *sqlite3WalFile(Wal *pWal){ return pWal->pWalFd; } + +/* +** Return the values required by sqlite3_wal_info(). +*/ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame){ + int rc = SQLITE_OK; + if( pWal ){ + *pnFrame = pWal->hdr.mxFrame; + *pnPrior = pWal->nPriorFrame; + } + return rc; +} #endif /* #ifndef SQLITE_OMIT_WAL */ Index: src/wal.h ================================================================== --- src/wal.h +++ src/wal.h @@ -32,11 +32,11 @@ # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 -# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalUndo(w,x,y,z) 0 # define sqlite3WalSavepoint(y,z) # define sqlite3WalSavepointUndo(y,z) 0 # define sqlite3WalFrames(u,v,w,x,y,z) 0 # define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 @@ -82,11 +82,11 @@ /* Obtain or release the WRITER lock. */ int sqlite3WalBeginWriteTransaction(Wal *pWal); int sqlite3WalEndWriteTransaction(Wal *pWal); /* Undo any frames written (but not committed) to the log */ -int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); +int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx, int); /* Return an integer that records the current (uncommitted) write ** position in the WAL */ void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); @@ -135,10 +135,19 @@ int sqlite3WalSnapshotRecover(Wal *pWal); int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot); void sqlite3WalSnapshotUnlock(Wal *pWal); #endif +#ifndef SQLITE_OMIT_CONCURRENT +/* Tell the wal layer that we want to commit a concurrent transaction */ +int sqlite3WalLockForCommit(Wal *pWal, PgHdr *pPg, Bitvec *pRead, Pgno*); + +/* Upgrade the state of the client to take into account changes written +** by other connections */ +int sqlite3WalUpgradeSnapshot(Wal *pWal); +#endif /* SQLITE_OMIT_CONCURRENT */ + #ifdef SQLITE_ENABLE_ZIPVFS /* If the WAL file is not empty, return the number of bytes of content ** stored in each frame (i.e. the db page-size when the WAL was created). */ int sqlite3WalFramesize(Wal *pWal); @@ -154,7 +163,10 @@ #ifdef SQLITE_USE_SEH int sqlite3WalSystemErrno(Wal*); #endif +/* sqlite3_wal_info() data */ +int sqlite3WalInfo(Wal *pWal, u32 *pnPrior, u32 *pnFrame); + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ ADDED test/bc_test1.c Index: test/bc_test1.c ================================================================== --- /dev/null +++ test/bc_test1.c @@ -0,0 +1,556 @@ +/* +** 2016-05-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + + +#include +#include +#include +#include "tt3_core.c" + +#ifdef USE_OSINST +# include "../src/test_osinst.c" +#else +# define vfslog_time() 0 +#endif + +typedef struct Config Config; +typedef struct ThreadCtx ThreadCtx; + +#define THREAD_TIME_INSERT 0 +#define THREAD_TIME_COMMIT 1 +#define THREAD_TIME_ROLLBACK 2 +#define THREAD_TIME_WRITER 3 +#define THREAD_TIME_CKPT 4 + +struct ThreadCtx { + Config *pConfig; + Sqlite *pDb; + Error *pErr; + sqlite3_int64 aTime[5]; +}; + +struct Config { + int nIPT; /* --inserts-per-transaction */ + int nThread; /* --threads */ + int nSecond; /* --seconds */ + int bMutex; /* --mutex */ + int nAutoCkpt; /* --autockpt */ + int bRm; /* --rm */ + int bClearCache; /* --clear-cache */ + int nMmap; /* mmap limit in MB */ + char *zFile; + int bOsinst; /* True to use osinst */ + + ThreadCtx *aCtx; /* Array of size nThread */ + + pthread_cond_t cond; + pthread_mutex_t mutex; + int nCondWait; /* Number of threads waiting on hCond */ + sqlite3_vfs *pVfs; +}; + + +typedef struct VfsWrapperFd VfsWrapperFd; +struct VfsWrapperFd { + sqlite3_file base; /* Base class */ + int bWriter; /* True if holding shm WRITER lock */ + int iTid; + Config *pConfig; + sqlite3_file *pFd; /* Underlying file descriptor */ +}; + +/* Methods of the wrapper VFS */ +static int vfsWrapOpen(sqlite3_vfs*, const char*, sqlite3_file*, int, int*); +static int vfsWrapDelete(sqlite3_vfs*, const char*, int); +static int vfsWrapAccess(sqlite3_vfs*, const char*, int, int*); +static int vfsWrapFullPathname(sqlite3_vfs*, const char *, int, char*); +static void *vfsWrapDlOpen(sqlite3_vfs*, const char*); +static void vfsWrapDlError(sqlite3_vfs*, int, char*); +static void (*vfsWrapDlSym(sqlite3_vfs*,void*, const char*))(void); +static void vfsWrapDlClose(sqlite3_vfs*, void*); +static int vfsWrapRandomness(sqlite3_vfs*, int, char*); +static int vfsWrapSleep(sqlite3_vfs*, int); +static int vfsWrapCurrentTime(sqlite3_vfs*, double*); +static int vfsWrapGetLastError(sqlite3_vfs*, int, char*); +static int vfsWrapCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int vfsWrapSetSystemCall(sqlite3_vfs*, const char*, sqlite3_syscall_ptr); +static sqlite3_syscall_ptr vfsWrapGetSystemCall(sqlite3_vfs*, const char*); +static const char *vfsWrapNextSystemCall(sqlite3_vfs*, const char*); + +/* Methods of wrapper sqlite3_io_methods object (see vfsWrapOpen()) */ +static int vfsWrapClose(sqlite3_file*); +static int vfsWrapRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int vfsWrapWrite(sqlite3_file*, const void*, int iAmt, sqlite3_int64); +static int vfsWrapTruncate(sqlite3_file*, sqlite3_int64 size); +static int vfsWrapSync(sqlite3_file*, int flags); +static int vfsWrapFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int vfsWrapLock(sqlite3_file*, int); +static int vfsWrapUnlock(sqlite3_file*, int); +static int vfsWrapCheckReservedLock(sqlite3_file*, int *pResOut); +static int vfsWrapFileControl(sqlite3_file*, int op, void *pArg); +static int vfsWrapSectorSize(sqlite3_file*); +static int vfsWrapDeviceCharacteristics(sqlite3_file*); +static int vfsWrapShmMap(sqlite3_file*, int iPg, int, int, void volatile**); +static int vfsWrapShmLock(sqlite3_file*, int offset, int n, int flags); +static void vfsWrapShmBarrier(sqlite3_file*); +static int vfsWrapShmUnmap(sqlite3_file*, int deleteFlag); +static int vfsWrapFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **); +static int vfsWrapUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +static int vfsWrapOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFd, + int flags, + int *fout +){ + static sqlite3_io_methods methods = { + 3, + vfsWrapClose, vfsWrapRead, vfsWrapWrite, + vfsWrapTruncate, vfsWrapSync, vfsWrapFileSize, + vfsWrapLock, vfsWrapUnlock, vfsWrapCheckReservedLock, + vfsWrapFileControl, vfsWrapSectorSize, vfsWrapDeviceCharacteristics, + vfsWrapShmMap, vfsWrapShmLock, vfsWrapShmBarrier, + vfsWrapShmUnmap, vfsWrapFetch, vfsWrapUnfetch + }; + + Config *pConfig = (Config*)pVfs->pAppData; + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + int rc; + + memset(pWrapper, 0, sizeof(VfsWrapperFd)); + if( flags & SQLITE_OPEN_MAIN_DB ){ + pWrapper->iTid = (int)sqlite3_uri_int64(zName, "tid", 0); + } + + pWrapper->pFd = (sqlite3_file*)&pWrapper[1]; + pWrapper->pConfig = pConfig; + rc = pConfig->pVfs->xOpen(pConfig->pVfs, zName, pWrapper->pFd, flags, fout); + if( rc==SQLITE_OK ){ + pWrapper->base.pMethods = &methods; + } + return rc; +} + +static int vfsWrapDelete(sqlite3_vfs *pVfs, const char *a, int b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDelete(pConfig->pVfs, a, b); +} +static int vfsWrapAccess(sqlite3_vfs *pVfs, const char *a, int b, int *c){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xAccess(pConfig->pVfs, a, b, c); +} +static int vfsWrapFullPathname(sqlite3_vfs *pVfs, const char *a, int b, char*c){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xFullPathname(pConfig->pVfs, a, b, c); +} +static void *vfsWrapDlOpen(sqlite3_vfs *pVfs, const char *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlOpen(pConfig->pVfs, a); +} +static void vfsWrapDlError(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlError(pConfig->pVfs, a, b); +} +static void (*vfsWrapDlSym(sqlite3_vfs *pVfs, void *a, const char *b))(void){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlSym(pConfig->pVfs, a, b); +} +static void vfsWrapDlClose(sqlite3_vfs *pVfs, void *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xDlClose(pConfig->pVfs, a); +} +static int vfsWrapRandomness(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xRandomness(pConfig->pVfs, a, b); +} +static int vfsWrapSleep(sqlite3_vfs *pVfs, int a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xSleep(pConfig->pVfs, a); +} +static int vfsWrapCurrentTime(sqlite3_vfs *pVfs, double *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xCurrentTime(pConfig->pVfs, a); +} +static int vfsWrapGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xGetLastError(pConfig->pVfs, a, b); +} +static int vfsWrapCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xCurrentTimeInt64(pConfig->pVfs, a); +} +static int vfsWrapSetSystemCall( + sqlite3_vfs *pVfs, + const char *a, + sqlite3_syscall_ptr b +){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xSetSystemCall(pConfig->pVfs, a, b); +} +static sqlite3_syscall_ptr vfsWrapGetSystemCall( + sqlite3_vfs *pVfs, + const char *a +){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xGetSystemCall(pConfig->pVfs, a); +} +static const char *vfsWrapNextSystemCall(sqlite3_vfs *pVfs, const char *a){ + Config *pConfig = (Config*)pVfs->pAppData; + return pConfig->pVfs->xNextSystemCall(pConfig->pVfs, a); +} + +static int vfsWrapClose(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + pWrapper->pFd->pMethods->xClose(pWrapper->pFd); + pWrapper->pFd = 0; + return SQLITE_OK; +} +static int vfsWrapRead(sqlite3_file *pFd, void *a, int b, sqlite3_int64 c){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xRead(pWrapper->pFd, a, b, c); +} +static int vfsWrapWrite( + sqlite3_file *pFd, + const void *a, int b, + sqlite3_int64 c +){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xWrite(pWrapper->pFd, a, b, c); +} +static int vfsWrapTruncate(sqlite3_file *pFd, sqlite3_int64 a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xTruncate(pWrapper->pFd, a); +} +static int vfsWrapSync(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xSync(pWrapper->pFd, a); +} +static int vfsWrapFileSize(sqlite3_file *pFd, sqlite3_int64 *a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFileSize(pWrapper->pFd, a); +} +static int vfsWrapLock(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xLock(pWrapper->pFd, a); +} +static int vfsWrapUnlock(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xUnlock(pWrapper->pFd, a); +} +static int vfsWrapCheckReservedLock(sqlite3_file *pFd, int *a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xCheckReservedLock(pWrapper->pFd, a); +} +static int vfsWrapFileControl(sqlite3_file *pFd, int a, void *b){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFileControl(pWrapper->pFd, a, b); +} +static int vfsWrapSectorSize(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xSectorSize(pWrapper->pFd); +} +static int vfsWrapDeviceCharacteristics(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xDeviceCharacteristics(pWrapper->pFd); +} +static int vfsWrapShmMap( + sqlite3_file *pFd, + int a, int b, int c, + void volatile **d +){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmMap(pWrapper->pFd, a, b, c, d); +} +static int vfsWrapShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + Config *pConfig = pWrapper->pConfig; + int bMutex = 0; + int rc; + + if( (offset==0 && n==1) + && (flags & SQLITE_SHM_LOCK) && (flags & SQLITE_SHM_EXCLUSIVE) + ){ + pthread_mutex_lock(&pConfig->mutex); + pWrapper->bWriter = 1; + bMutex = 1; + if( pWrapper->iTid ){ + sqlite3_int64 t = vfslog_time(); + pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] -= t; + } + } + + rc = pWrapper->pFd->pMethods->xShmLock(pWrapper->pFd, offset, n, flags); + + if( (rc!=SQLITE_OK && bMutex) + || (offset==0 && (flags & SQLITE_SHM_UNLOCK) && pWrapper->bWriter) + ){ + assert( pWrapper->bWriter ); + pthread_mutex_unlock(&pConfig->mutex); + pWrapper->bWriter = 0; + if( pWrapper->iTid ){ + sqlite3_int64 t = vfslog_time(); + pConfig->aCtx[pWrapper->iTid-1].aTime[THREAD_TIME_WRITER] += t; + } + } + + return rc; +} +static void vfsWrapShmBarrier(sqlite3_file *pFd){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmBarrier(pWrapper->pFd); +} +static int vfsWrapShmUnmap(sqlite3_file *pFd, int a){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xShmUnmap(pWrapper->pFd, a); +} +static int vfsWrapFetch(sqlite3_file *pFd, sqlite3_int64 a, int b, void **c){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xFetch(pWrapper->pFd, a, b, c); +} +static int vfsWrapUnfetch(sqlite3_file *pFd, sqlite3_int64 a, void *b){ + VfsWrapperFd *pWrapper = (VfsWrapperFd*)pFd; + return pWrapper->pFd->pMethods->xUnfetch(pWrapper->pFd, a, b); +} + +static void create_vfs(Config *pConfig){ + static sqlite3_vfs vfs = { + 3, 0, 0, 0, "wrapper", 0, + vfsWrapOpen, vfsWrapDelete, vfsWrapAccess, + vfsWrapFullPathname, vfsWrapDlOpen, vfsWrapDlError, + vfsWrapDlSym, vfsWrapDlClose, vfsWrapRandomness, + vfsWrapSleep, vfsWrapCurrentTime, vfsWrapGetLastError, + vfsWrapCurrentTimeInt64, vfsWrapSetSystemCall, vfsWrapGetSystemCall, + vfsWrapNextSystemCall + }; + sqlite3_vfs *pVfs; + + pVfs = sqlite3_vfs_find(0); + vfs.mxPathname = pVfs->mxPathname; + vfs.szOsFile = pVfs->szOsFile + sizeof(VfsWrapperFd); + vfs.pAppData = (void*)pConfig; + pConfig->pVfs = pVfs; + + sqlite3_vfs_register(&vfs, 1); +} + + +/* +** Wal hook used by connections in thread_main(). +*/ +static int thread_wal_hook( + void *pArg, /* Pointer to ThreadCtx object */ + sqlite3 *db, + const char *zDb, + int nFrame +){ + ThreadCtx *pCtx = (ThreadCtx*)pArg; + Config *pConfig = pCtx->pConfig; + + if( pConfig->nAutoCkpt && nFrame>=pConfig->nAutoCkpt ){ + pCtx->aTime[THREAD_TIME_CKPT] -= vfslog_time(); + pthread_mutex_lock(&pConfig->mutex); + if( pConfig->nCondWait>=0 ){ + pConfig->nCondWait++; + if( pConfig->nCondWait==pConfig->nThread ){ + execsql(pCtx->pErr, pCtx->pDb, "PRAGMA wal_checkpoint"); + pthread_cond_broadcast(&pConfig->cond); + }else{ + pthread_cond_wait(&pConfig->cond, &pConfig->mutex); + } + pConfig->nCondWait--; + } + pthread_mutex_unlock(&pConfig->mutex); + pCtx->aTime[THREAD_TIME_CKPT] += vfslog_time(); + } + + return SQLITE_OK; +} + + +static char *thread_main(int iTid, void *pArg){ + Config *pConfig = (Config*)pArg; + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int nAttempt = 0; /* Attempted transactions */ + int nCommit = 0; /* Successful transactions */ + int j; + ThreadCtx *pCtx = &pConfig->aCtx[iTid-1]; + char *zUri = 0; + +#ifdef USE_OSINST + char *zOsinstName = 0; + char *zLogName = 0; + if( pConfig->bOsinst ){ + zOsinstName = sqlite3_mprintf("osinst%d", iTid); + zLogName = sqlite3_mprintf("bc_test1.log.%d.%d", (int)getpid(), iTid); + zUri = sqlite3_mprintf( + "file:%s?vfs=%s&tid=%d", pConfig->zFile, zOsinstName, iTid + ); + sqlite3_vfslog_new(zOsinstName, 0, zLogName); + opendb(&err, &db, zUri, 0); + }else +#endif + { + zUri = sqlite3_mprintf("file:%s?tid=%d", pConfig->zFile, iTid); + opendb(&err, &db, zUri, 0); + } + + sqlite3_busy_handler(db.db, 0, 0); + sql_script_printf(&err, &db, + "PRAGMA wal_autocheckpoint = 0;" + "PRAGMA synchronous = 0;" + "PRAGMA mmap_size = %lld;", + (i64)(pConfig->nMmap) * 1024 * 1024 + ); + + pCtx->pConfig = pConfig; + pCtx->pErr = &err; + pCtx->pDb = &db; + sqlite3_wal_hook(db.db, thread_wal_hook, (void*)pCtx); + + while( !timetostop(&err) ){ + execsql(&err, &db, "BEGIN CONCURRENT"); + + pCtx->aTime[THREAD_TIME_INSERT] -= vfslog_time(); + for(j=0; jnIPT; j++){ + execsql(&err, &db, + "INSERT INTO t1 VALUES" + "(randomblob(10), randomblob(20), randomblob(30), randomblob(200))" + ); + } + pCtx->aTime[THREAD_TIME_INSERT] += vfslog_time(); + + pCtx->aTime[THREAD_TIME_COMMIT] -= vfslog_time(); + execsql(&err, &db, "COMMIT"); + pCtx->aTime[THREAD_TIME_COMMIT] += vfslog_time(); + + pCtx->aTime[THREAD_TIME_ROLLBACK] -= vfslog_time(); + nAttempt++; + if( err.rc==SQLITE_OK ){ + nCommit++; + }else{ + clear_error(&err, SQLITE_BUSY); + execsql(&err, &db, "ROLLBACK"); + } + pCtx->aTime[THREAD_TIME_ROLLBACK] += vfslog_time(); + + if( pConfig->bClearCache ){ + sqlite3_db_release_memory(db.db); + } + } + + closedb(&err, &db); + +#ifdef USE_OSINST + if( pConfig->bOsinst ){ + sqlite3_vfslog_finalize(zOsinstName); + sqlite3_free(zOsinstName); + sqlite3_free(zLogName); + } +#endif + sqlite3_free(zUri); + + pthread_mutex_lock(&pConfig->mutex); + pConfig->nCondWait = -1; + pthread_cond_broadcast(&pConfig->cond); + pthread_mutex_unlock(&pConfig->mutex); + + return sqlite3_mprintf("commits: %d/%d insert: %dms" + " commit: %dms" + " rollback: %dms" + " writer: %dms" + " checkpoint: %dms", + nCommit, nAttempt, + (int)(pCtx->aTime[THREAD_TIME_INSERT]/1000), + (int)(pCtx->aTime[THREAD_TIME_COMMIT]/1000), + (int)(pCtx->aTime[THREAD_TIME_ROLLBACK]/1000), + (int)(pCtx->aTime[THREAD_TIME_WRITER]/1000), + (int)(pCtx->aTime[THREAD_TIME_CKPT]/1000) + ); +} + +int main(int argc, const char **argv){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + Threadset threads = {0}; /* Test threads */ + Config conf = {5, 3, 5}; + int i; + + CmdlineArg apArg[] = { + { "-seconds", CMDLINE_INT, offsetof(Config, nSecond) }, + { "-inserts", CMDLINE_INT, offsetof(Config, nIPT) }, + { "-threads", CMDLINE_INT, offsetof(Config, nThread) }, + { "-mutex", CMDLINE_BOOL, offsetof(Config, bMutex) }, + { "-rm", CMDLINE_BOOL, offsetof(Config, bRm) }, + { "-autockpt",CMDLINE_INT, offsetof(Config, nAutoCkpt) }, + { "-mmap", CMDLINE_INT, offsetof(Config, nMmap) }, + { "-clear-cache", CMDLINE_BOOL, offsetof(Config, bClearCache) }, + { "-file", CMDLINE_STRING, offsetof(Config, zFile) }, + { "-osinst", CMDLINE_BOOL, offsetof(Config, bOsinst) }, + { 0, 0, 0 } + }; + + conf.nAutoCkpt = 1000; + cmdline_process(apArg, argc, argv, (void*)&conf); + if( err.rc==SQLITE_OK ){ + char *z = cmdline_construct(apArg, (void*)&conf); + printf("With: %s\n", z); + sqlite3_free(z); + } + if( conf.zFile==0 ){ + conf.zFile = "xyz.db"; + } + + /* Create the special VFS - "wrapper". And the mutex and condition + ** variable. */ + create_vfs(&conf); + pthread_mutex_init(&conf.mutex, 0); + pthread_cond_init(&conf.cond, 0); + + conf.aCtx = sqlite3_malloc(sizeof(ThreadCtx) * conf.nThread); + memset(conf.aCtx, 0, sizeof(ThreadCtx) * conf.nThread); + + /* Ensure the schema has been created */ + opendb(&err, &db, conf.zFile, conf.bRm); + sql_script(&err, &db, + "PRAGMA journal_mode = wal;" + "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID;" + "CREATE INDEX IF NOT EXISTS t1b ON t1(b);" + "CREATE INDEX IF NOT EXISTS t1c ON t1(c);" + ); + + setstoptime(&err, conf.nSecond*1000); + if( conf.nThread==1 ){ + char *z = thread_main(1, (void*)&conf); + printf("Thread 0 says: %s\n", (z==0 ? "..." : z)); + fflush(stdout); + }else{ + for(i=0; i$nPg + } {1} + + do_test 1.$tn.7 { + sql2 { PRAGMA integrity_check } + } {ok} + + do_test 1.$tn.8 { + sql1 { + BEGIN CONCURRENT; + CREATE TABLE t4(a, b); + } + sql2 { + INSERT INTO t1 VALUES(2, 2); + } + list [catch { sql1 COMMIT } msg] $msg + } {1 {database is locked}} + sql1 ROLLBACK + + do_test 1.$tn.9 { + sql1 { + BEGIN CONCURRENT; + CREATE TEMP TABLE t5(a, b); + INSERT INTO t2 VALUES('x', 'x'); + } + sql2 { + INSERT INTO t1 VALUES(3, 3); + CREATE TEMP TABLE t1(x, y); + } + sql1 COMMIT + } {} +} + + + +finish_test + + ADDED test/concurrent9.test Index: test/concurrent9.test ================================================================== --- /dev/null +++ test/concurrent9.test @@ -0,0 +1,55 @@ +# 2023 January 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix concurrent9 + +do_execsql_test 1.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1), (2); + CREATE TABLE t2(y); + INSERT INTO t2 VALUES('a'), ('b'); + PRAGMA journal_mode = wal; +} {wal} + +db close + +#------------------------------------------------------------------------- +# Fix a problem that may occur if a BEGIN CONCURRENT transaction is +# started when the wal file is completely empty and committed after +# it has been initialized by some other connection. +# +sqlite3 db test.db +sqlite3 db2 test.db + +do_execsql_test -db db 1.1 { + BEGIN CONCURRENT; + INSERT INTO t2 VALUES('c'); +} + +do_execsql_test -db db2 1.2 { + INSERT INTO t1 VALUES(3); +} + +do_execsql_test -db db 1.3 { + COMMIT; +} + +do_execsql_test -db db2 1.4 { + SELECT * FROM t1; + SELECT * FROM t2; +} {1 2 3 a b c} + +finish_test + Index: test/corruptN.test ================================================================== --- test/corruptN.test +++ test/corruptN.test @@ -139,19 +139,10 @@ | 0: 0a 00 00 00 02 0f f5 00 0f fb 0f f5 00 00 00 00 ................ | 4080: 00 00 00 00 00 05 03 01 01 0d 02 04 03 00 00 00 ................ | end c-b92b.txt.db }]} {} -# This test only works with the legacy RC4 PRNG -if 0 { - prng_seed 0 db - do_catchsql_test 2.1 { - SELECT count(*) FROM sqlite_schema; - WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) - INSERT INTO t1(a) SELECT randomblob(null) FROM c; - } {1 {database disk image is malformed}} -} reset_db if {![info exists ::G(perm:presql)]} { do_execsql_test 3.0 { CREATE TABLE t1(x INTEGER PRIMARY KEY AUTOINCREMENT, y); Index: test/fts3corrupt4.test ================================================================== --- test/fts3corrupt4.test +++ test/fts3corrupt4.test @@ -4394,19 +4394,21 @@ do_catchsql_test 25.4 { WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x%1 FROM c WHERE 599237 +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* +** The "Set Error Line" macro. +*/ +#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) + +/* Database functions */ +#define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) +#define closedb(y,z) (SEL(y), closedb_x(y,z)) + +/* Functions to execute SQL */ +#define sql_script(x,y,z) (SEL(x), sql_script_x(x,y,z)) +#define integrity_check(x,y) (SEL(x), integrity_check_x(x,y)) +#define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) +#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) +#define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) +#define sql_script_printf(x,y,z,...) ( \ + SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__) \ +) + +/* Thread functions */ +#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) +#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) + +/* Timer functions */ +#define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) +#define timetostop(z) (SEL(z), timetostop_x(z)) + +/* Report/clear errors. */ +#define test_error(z, ...) test_error_x(z, sqlite3_mprintf(__VA_ARGS__)) +#define clear_error(y,z) clear_error_x(y, z) + +/* File-system operations */ +#define filesize(y,z) (SEL(y), filesize_x(y,z)) +#define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) + +#define PTR2INT(x) ((int)((intptr_t)x)) +#define INT2PTR(x) ((void*)((intptr_t)x)) + +/* +** End of test code/infrastructure interface macros. +*************************************************************************/ + + +/************************************************************************ +** Start of command line processing utilities. +*/ +#define CMDLINE_INT 1 +#define CMDLINE_BOOL 2 +#define CMDLINE_STRING 3 + +typedef struct CmdlineArg CmdlineArg; +struct CmdlineArg { + const char *zSwitch; + int eType; + int iOffset; +}; + +static void cmdline_error(const char *zFmt, ...){ + va_list ap; /* ... arguments */ + char *zMsg = 0; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + fprintf(stderr, "%s\n", zMsg); + sqlite3_free(zMsg); + va_end(ap); + exit(-1); +} + +static void cmdline_usage(const char *zPrg, CmdlineArg *apArg){ + int i; + fprintf(stderr, "Usage: %s SWITCHES\n", zPrg); + fprintf(stderr, "\n"); + fprintf(stderr, "where switches are\n"); + for(i=0; apArg[i].zSwitch; i++){ + const char *zExtra = ""; + switch( apArg[i].eType ){ + case CMDLINE_STRING: zExtra = "STRING"; break; + case CMDLINE_INT: zExtra = "N"; break; + case CMDLINE_BOOL: zExtra = ""; break; + default: + zExtra = "???"; + break; + } + fprintf(stderr, " %s %s\n", apArg[i].zSwitch, zExtra); + } + fprintf(stderr, "\n"); + exit(-2); +} + +static char *cmdline_construct(CmdlineArg *apArg, void *pObj){ + unsigned char *p = (unsigned char*)pObj; + char *zRet = 0; + int iArg; + + for(iArg=0; apArg[iArg].zSwitch; iArg++){ + const char *zSpace = (zRet ? " " : ""); + CmdlineArg *pArg = &apArg[iArg]; + + switch( pArg->eType ){ + case CMDLINE_STRING: { + char *zVal = *(char**)(p + pArg->iOffset); + if( zVal ){ + zRet = sqlite3_mprintf("%z%s%s %s", zRet, zSpace, pArg->zSwitch,zVal); + } + break; + }; + + case CMDLINE_INT: { + zRet = sqlite3_mprintf("%z%s%s %d", zRet, zSpace, pArg->zSwitch, + *(int*)(p + pArg->iOffset) + ); + break; + }; + + case CMDLINE_BOOL: + if( *(int*)(p + pArg->iOffset) ){ + zRet = sqlite3_mprintf("%z%s%s", zRet, zSpace, pArg->zSwitch); + } + break; + + default: + zRet = sqlite3_mprintf("%z%s%s ???", zRet, zSpace, pArg->zSwitch); + } + } + + return zRet; +} + +static void cmdline_process( + CmdlineArg *apArg, + int argc, + const char **argv, + void *pObj +){ + int i; + int iArg; + unsigned char *p = (unsigned char*)pObj; + + for(i=1; i=0 ){ + cmdline_error("ambiguous switch: %s", z); + } + iOpt = iArg; + switch( apArg[iArg].eType ){ + case CMDLINE_INT: + i++; + if( i==argc ){ + cmdline_error("option requires an argument: %s", z); + } + *(int*)(p + apArg[iArg].iOffset) = atoi(argv[i]); + break; + + case CMDLINE_STRING: + i++; + if( i==argc ){ + cmdline_error("option requires an argument: %s", z); + } + *(char**)(p + apArg[iArg].iOffset) = sqlite3_mprintf("%s", argv[i]); + break; + + case CMDLINE_BOOL: + *(int*)(p + apArg[iArg].iOffset) = 1; + break; + + default: + assert( 0 ); + cmdline_error("internal error"); + return; + } + } + } + + if( iOpt<0 ){ + cmdline_usage(argv[0], apArg); + } + } +} + +/* +** End of command line processing utilities. +*************************************************************************/ + + +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ + +/* + * If compiled on a machine that doesn't have a 32-bit integer, + * you just set "uint32" to the appropriate datatype for an + * unsigned 32-bit integer. For example: + * + * cc -Duint32='unsigned long' md5.c + * + */ +#ifndef uint32 +# define uint32 unsigned int +#endif + +struct MD5Context { + int isInit; + uint32 buf[4]; + uint32 bits[2]; + union { + unsigned char in[64]; + uint32 in32[16]; + } u; +}; +typedef struct MD5Context MD5Context; + +/* + * Note: this code is harmless on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32 t; + do { + t = (uint32)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32 *)buf = t; + buf += 4; + } while (--longs); +} +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32 buf[4], const uint32 in[16]){ + register uint32 a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +static void MD5Init(MD5Context *ctx){ + ctx->isInit = 1; + ctx->buf[0] = 0x67452301; + ctx->buf[1] = 0xefcdab89; + ctx->buf[2] = 0x98badcfe; + ctx->buf[3] = 0x10325476; + ctx->bits[0] = 0; + ctx->bits[1] = 0; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +static +void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ + uint32 t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->u.in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->u.in, buf, 64); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->u.in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +static void MD5Final(unsigned char digest[16], MD5Context *ctx){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->u.in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->u.in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->u.in, 14); + + /* Append length in bits and transform */ + ctx->u.in32[14] = ctx->bits[0]; + ctx->u.in32[15] = ctx->bits[1]; + + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ +} + +/* +** Convert a 128-bit MD5 digest into a 32-digit base-16 number. +*/ +static void MD5DigestToBase16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + +/* +** During testing, the special md5sum() aggregate function is available. +** inside SQLite. The following routines implement that function. +*/ +static void md5step(sqlite3_context *context, int argc, sqlite3_value **argv){ + MD5Context *p; + int i; + if( argc<1 ) return; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + MD5Init(p); + } + for(i=0; izErr); + p->zErr = 0; + p->rc = 0; +} + +static void print_err(Error *p){ + if( p->rc!=SQLITE_OK ){ + int isWarn = 0; + if( p->rc==SQLITE_SCHEMA ) isWarn = 1; + if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1; + printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error", + p->rc, p->zErr, p->iLine); + if( !isWarn ) nGlobalErr++; + fflush(stdout); + } +} + +static void print_and_free_err(Error *p){ + print_err(p); + free_err(p); +} + +static void system_error(Error *pErr, int iSys){ + pErr->rc = iSys; + pErr->zErr = (char *)sqlite3_malloc(512); + strerror_r(iSys, pErr->zErr, 512); + pErr->zErr[511] = '\0'; +} + +static void sqlite_error( + Error *pErr, + Sqlite *pDb, + const char *zFunc +){ + pErr->rc = sqlite3_errcode(pDb->db); + pErr->zErr = sqlite3_mprintf( + "sqlite3_%s() - %s (%d)", zFunc, sqlite3_errmsg(pDb->db), + sqlite3_extended_errcode(pDb->db) + ); +} + +static void test_error_x( + Error *pErr, + char *zErr +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = 1; + pErr->zErr = zErr; + }else{ + sqlite3_free(zErr); + } +} + +static void clear_error_x( + Error *pErr, + int rc +){ + if( pErr->rc==rc ){ + pErr->rc = SQLITE_OK; + sqlite3_free(pErr->zErr); + pErr->zErr = 0; + } +} + +static int busyhandler(void *pArg, int n){ + usleep(10*1000); + return 1; +} + +static void opendb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* OUT: Database handle */ + const char *zFile, /* Database file name */ + int bDelete /* True to delete db file before opening */ +){ + if( pErr->rc==SQLITE_OK ){ + int rc; + int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; + if( bDelete ) unlink(zFile); + rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0); + if( rc ){ + sqlite_error(pErr, pDb, "open"); + sqlite3_close(pDb->db); + pDb->db = 0; + }else{ + sqlite3_create_function( + pDb->db, "md5sum", -1, SQLITE_UTF8, 0, 0, md5step, md5finalize + ); + sqlite3_busy_handler(pDb->db, busyhandler, 0); + sqlite3_exec(pDb->db, "PRAGMA synchronous=OFF", 0, 0, 0); + } + } +} + +static void closedb_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* OUT: Database handle */ +){ + int rc; + int i; + Statement *pIter; + Statement *pNext; + for(pIter=pDb->pCache; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_finalize(pIter->pStmt); + sqlite3_free(pIter); + } + for(i=0; inText; i++){ + sqlite3_free(pDb->aText[i]); + } + sqlite3_free(pDb->aText); + rc = sqlite3_close(pDb->db); + if( rc && pErr->rc==SQLITE_OK ){ + pErr->zErr = sqlite3_mprintf("%s", sqlite3_errmsg(pDb->db)); + } + memset(pDb, 0, sizeof(Sqlite)); +} + +static void sql_script_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL script to execute */ +){ + if( pErr->rc==SQLITE_OK ){ + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + } +} + +static void sql_script_printf_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zFormat, /* SQL printf format string */ + ... /* Printf args */ +){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + if( pErr->rc==SQLITE_OK ){ + char *zSql = sqlite3_vmprintf(zFormat, ap); + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + sqlite3_free(zSql); + } + va_end(ap); +} + +static Statement *getSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zSql /* SQL statement */ +){ + Statement *pRet; + int rc; + + for(pRet=pDb->pCache; pRet; pRet=pRet->pNext){ + if( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ){ + return pRet; + } + } + + pRet = sqlite3_malloc(sizeof(Statement)); + rc = sqlite3_prepare_v2(pDb->db, zSql, -1, &pRet->pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite_error(pErr, pDb, "prepare_v2"); + return 0; + } + assert( 0==strcmp(sqlite3_sql(pRet->pStmt), zSql) ); + + pRet->pNext = pDb->pCache; + pDb->pCache = pRet; + return pRet; +} + +static sqlite3_stmt *getAndBindSqlStatement( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + va_list ap /* SQL followed by parameters */ +){ + Statement *pStatement; /* The SQLite statement wrapper */ + sqlite3_stmt *pStmt; /* The SQLite statement to return */ + int i; /* Used to iterate through parameters */ + + pStatement = getSqlStatement(pErr, pDb, va_arg(ap, const char *)); + if( !pStatement ) return 0; + pStmt = pStatement->pStmt; + for(i=1; i<=sqlite3_bind_parameter_count(pStmt); i++){ + const char *zName = sqlite3_bind_parameter_name(pStmt, i); + void * pArg = va_arg(ap, void*); + + switch( zName[1] ){ + case 'i': + sqlite3_bind_int64(pStmt, i, *(i64 *)pArg); + break; + + default: + pErr->rc = 1; + pErr->zErr = sqlite3_mprintf("Cannot discern type: \"%s\"", zName); + pStmt = 0; + break; + } + } + + return pStmt; +} + +static i64 execsql_i64_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + ... /* SQL and pointers to parameter values */ +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + va_start(ap, pDb); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + iRet = sqlite3_column_int64(pStmt, 0); + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + return iRet; +} + +static char * execsql_text_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + int iSlot, /* Db handle slot to store text in */ + ... /* SQL and pointers to parameter values */ +){ + char *zRet = 0; + + if( iSlot>=pDb->nText ){ + int nByte = sizeof(char *)*(iSlot+1); + pDb->aText = (char **)sqlite3_realloc(pDb->aText, nByte); + memset(&pDb->aText[pDb->nText], 0, sizeof(char*)*(iSlot+1-pDb->nText)); + pDb->nText = iSlot+1; + } + + if( pErr->rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; /* SQL statement to execute */ + va_list ap; /* ... arguments */ + va_start(ap, iSlot); + pStmt = getAndBindSqlStatement(pErr, pDb, ap); + if( pStmt ){ + int first = 1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( first && sqlite3_column_count(pStmt)>0 ){ + zRet = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); + sqlite3_free(pDb->aText[iSlot]); + pDb->aText[iSlot] = zRet; + } + first = 0; + } + if( SQLITE_OK!=sqlite3_reset(pStmt) ){ + sqlite_error(pErr, pDb, "reset"); + } + } + va_end(ap); + } + + return zRet; +} + +static void integrity_check_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb /* Database handle */ +){ + if( pErr->rc==SQLITE_OK ){ + Statement *pStatement; /* Statement to execute */ + char *zErr = 0; /* Integrity check error */ + + pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); + if( pStatement ){ + sqlite3_stmt *pStmt = pStatement->pStmt; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *z = (const char*)sqlite3_column_text(pStmt, 0); + if( strcmp(z, "ok") ){ + if( zErr==0 ){ + zErr = sqlite3_mprintf("%s", z); + }else{ + zErr = sqlite3_mprintf("%z\n%s", zErr, z); + } + } + } + sqlite3_reset(pStmt); + + if( zErr ){ + pErr->zErr = zErr; + pErr->rc = 1; + } + } + } +} + +static void *launch_thread_main(void *pArg){ + Thread *p = (Thread *)pArg; + return (void *)p->xProc(p->iTid, p->pArg); +} + +static void launch_thread_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads, /* Thread set */ + char *(*xProc)(int, void*), /* Proc to run */ + void *pArg /* Argument passed to thread proc */ +){ + if( pErr->rc==SQLITE_OK ){ + int iTid = ++pThreads->iMaxTid; + Thread *p; + int rc; + + p = (Thread *)sqlite3_malloc(sizeof(Thread)); + memset(p, 0, sizeof(Thread)); + p->iTid = iTid; + p->pArg = pArg; + p->xProc = xProc; + + rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); + if( rc!=0 ){ + system_error(pErr, rc); + sqlite3_free(p); + }else{ + p->pNext = pThreads->pThread; + pThreads->pThread = p; + } + } +} + +static void join_all_threads_x( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads /* Thread set */ +){ + Thread *p; + Thread *pNext; + for(p=pThreads->pThread; p; p=pNext){ + void *ret; + pNext = p->pNext; + int rc; + rc = pthread_join(p->tid, &ret); + if( rc!=0 ){ + if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); + }else{ + printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); + fflush(stdout); + } + sqlite3_free(p); + } + pThreads->pThread = 0; +} + +static i64 filesize_x( + Error *pErr, + const char *zFile +){ + i64 iRet = 0; + if( pErr->rc==SQLITE_OK ){ + struct stat sStat; + if( stat(zFile, &sStat) ){ + iRet = -1; + }else{ + iRet = sStat.st_size; + } + } + return iRet; +} + +static void filecopy_x( + Error *pErr, + const char *zFrom, + const char *zTo +){ + if( pErr->rc==SQLITE_OK ){ + i64 nByte = filesize_x(pErr, zFrom); + if( nByte<0 ){ + test_error_x(pErr, sqlite3_mprintf("no such file: %s", zFrom)); + }else{ + i64 iOff; + char aBuf[1024]; + int fd1; + int fd2; + unlink(zTo); + + fd1 = open(zFrom, O_RDONLY); + if( fd1<0 ){ + system_error(pErr, errno); + return; + } + fd2 = open(zTo, O_RDWR|O_CREAT|O_EXCL, 0644); + if( fd2<0 ){ + system_error(pErr, errno); + close(fd1); + return; + } + + iOff = 0; + while( iOffnByte ){ + nCopy = nByte - iOff; + } + if( nCopy!=read(fd1, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + if( nCopy!=write(fd2, aBuf, nCopy) ){ + system_error(pErr, errno); + break; + } + iOff += nCopy; + } + + close(fd1); + close(fd2); + } + } +} + +/* +** Used by setstoptime() and timetostop(). +*/ +static double timelimit = 0.0; + +static double currentTime(void){ + double t; + static sqlite3_vfs *pTimelimitVfs = 0; + if( pTimelimitVfs==0 ) pTimelimitVfs = sqlite3_vfs_find(0); + if( pTimelimitVfs->iVersion>=2 && pTimelimitVfs->xCurrentTimeInt64!=0 ){ + sqlite3_int64 tm; + pTimelimitVfs->xCurrentTimeInt64(pTimelimitVfs, &tm); + t = tm/86400000.0; + }else{ + pTimelimitVfs->xCurrentTime(pTimelimitVfs, &t); + } + return t; +} + +static void setstoptime_x( + Error *pErr, /* IN/OUT: Error code */ + int nMs /* Milliseconds until "stop time" */ +){ + if( pErr->rc==SQLITE_OK ){ + double t = currentTime(); + timelimit = t + ((double)nMs)/(1000.0*60.0*60.0*24.0); + } +} + +static int timetostop_x( + Error *pErr /* IN/OUT: Error code */ +){ + int ret = 1; + if( pErr->rc==SQLITE_OK ){ + double t = currentTime(); + ret = (t >= timelimit); + } + return ret; +} + Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -32,47 +32,10 @@ ifcapable !dirsync { incr sqlite_sync_count $adj } } } - -proc set_tvfs_hdr {file args} { - - # Set $nHdr to the number of bytes in the wal-index header: - set nHdr 48 - set nInt [expr {$nHdr/4}] - - if {[llength $args]>2} { - error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"} - } - - set blob [tvfs shm $file] - if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i} - - if {[llength $args]} { - set ia [lindex $args 0] - set ib $ia - if {[llength $args]==2} { - set ib [lindex $args 1] - } - binary scan $blob a[expr $nHdr*2]a* dummy tail - set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail] - tvfs shm $file $blob - } - - binary scan $blob ${fmt}${nInt} ints - return $ints -} - -proc incr_tvfs_hdr {file idx incrval} { - set ints [set_tvfs_hdr $file] - set v [lindex $ints $idx] - incr v $incrval - lset ints $idx $v - set_tvfs_hdr $file $ints -} - #------------------------------------------------------------------------- # Test case wal2-1.*: # # Set up a small database containing a single table. The database is not Index: test/wal_common.tcl ================================================================== --- test/wal_common.tcl +++ test/wal_common.tcl @@ -87,7 +87,47 @@ set c2 0 wal_cksum_intlist c1 c2 [lrange $hdr 0 9] lset hdr 10 $c1 lset hdr 11 $c2 } + +# This command assumes that $file is the name of a database file opened +# in wal mode using a [testvfs] VFS. It returns a list of the 12 32-bit +# integers that make up the wal-index-header for the named file. +# +proc set_tvfs_hdr {file args} { + + # Set $nHdr to the number of bytes in the wal-index header: + set nHdr 48 + set nInt [expr {$nHdr/4}] + + if {[llength $args]>2} { + error {wrong # args: should be "set_tvfs_hdr fileName ?val1? ?val2?"} + } + + set blob [tvfs shm $file] + if {$::tcl_platform(byteOrder)=="bigEndian"} {set fmt I} {set fmt i} + + if {[llength $args]} { + set ia [lindex $args 0] + set ib $ia + if {[llength $args]==2} { + set ib [lindex $args 1] + } + binary scan $blob a[expr $nHdr*2]a* dummy tail + set blob [binary format ${fmt}${nInt}${fmt}${nInt}a* $ia $ib $tail] + tvfs shm $file $blob + } + + binary scan $blob ${fmt}${nInt} ints + return $ints +} + +proc incr_tvfs_hdr {file idx incrval} { + set ints [set_tvfs_hdr $file] + set v [lindex $ints $idx] + incr v $incrval + lset ints $idx $v + set_tvfs_hdr $file $ints +} Index: tool/mkpragmatab.tcl ================================================================== --- tool/mkpragmatab.tcl +++ tool/mkpragmatab.tcl @@ -113,10 +113,16 @@ TYPE: FLAG ARG: SQLITE_VdbeEQP IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) IF: defined(SQLITE_DEBUG) + NAME: noop_update + TYPE: FLAG + ARG: SQLITE_NoopUpdate + IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) + IF: defined(SQLITE_ENABLE_NOOP_UPDATE) + NAME: ignore_check_constraints TYPE: FLAG ARG: SQLITE_IgnoreChecks IF: !defined(SQLITE_OMIT_FLAG_PRAGMAS) IF: !defined(SQLITE_OMIT_CHECK)