Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -62,11 +62,11 @@ # LIBREADLINE = @TARGET_READLINE_LIBS@ # Should the database engine be compiled threadsafe # -TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@ +#TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@ # Any target libraries which libsqlite must be linked against # TLIBS = @LIBS@ $(LIBS) @@ -116,11 +116,11 @@ # TCLSH_CMD = @TCLSH_CMD@ # Where do we want to install the tcl plugin # -TCLLIBDIR = @TCLLIBDIR@ +TCLLIBDIR = /System/Library/Tcl # The suffix used on shared libraries. Ex: ".dll", ".so", ".dylib" # SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@ @@ -351,10 +351,13 @@ $(TOP)/ext/icu/icu.c SRC += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c \ $(TOP)/ext/rtree/geopoly.c +SRC += \ + $(TOP)/ext/sqlrr/sqlrr.h \ + $(TOP)/ext/sqlrr/sqlrr.c SRC += \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/userauth/userauth.c \ @@ -568,10 +571,21 @@ $(TOP)/ext/icu/sqliteicu.h EXTHDR += \ $(TOP)/ext/rtree/sqlite3rtree.h EXTHDR += \ $(TOP)/ext/userauth/sqlite3userauth.h +EXTHDR += \ + $(TOP)/ext/sqlrr/sqlrr.h + +# If using the amalgamation, use sqlite3.c directly to build the test +# fixture. Otherwise link against libsqlite3.la. (This distinction is +# necessary because the test fixture requires non-API symbols which are +# hidden when the library is built via the amalgamation). +# +TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la +TESTFIXTURE_SRC1 = sqlite3.c +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) # executables needed for testing # TESTPROGS = \ testfixture$(TEXE) \ @@ -638,13 +652,12 @@ ${ALLOWRELEASE} -rpath "$(libdir)" -version-info "8:6:8" libtclsqlite3.la: tclsqlite.lo libsqlite3.la $(LTLINK) -no-undefined -o $@ tclsqlite.lo \ libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \ - -rpath "$(TCLLIBDIR)" \ - -version-info "8:6:8" \ - -avoid-version + -rpath "$(TCLLIBDIR)/sqlite3" \ + -version-info "8:6:8" sqlite3$(TEXE): shell.c sqlite3.c $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ shell.c sqlite3.c \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" @@ -1418,14 +1431,14 @@ $(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir) pkgIndex.tcl: echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@ tcl_install: lib_install libtclsqlite3.la pkgIndex.tcl - $(INSTALL) -d $(DESTDIR)$(TCLLIBDIR) - $(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR) - rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a - $(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR) + $(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)/sqlite3 + $(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/sqlite3 + rm -f $(DESTDIR)$(TCLLIBDIR)/sqlite3/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/sqlite3/libtclsqlite3.a + $(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)/sqlite3 clean: rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la rm -f sqlite3.h opcodes.* rm -rf .libs .deps Index: ext/fts5/test/fts5misc.test ================================================================== --- ext/fts5/test/fts5misc.test +++ ext/fts5/test/fts5misc.test @@ -322,6 +322,5 @@ do_execsql_test 12.3 { SELECT * FROM t2 JOIN ft USING (ft) } {3 4 b b} finish_test - Index: ext/rtree/rtree.c ================================================================== --- ext/rtree/rtree.c +++ ext/rtree/rtree.c @@ -80,10 +80,11 @@ #endif #include #include #include +#include /* The following macro is used to suppress compiler warnings. */ #ifndef UNUSED_PARAMETER # define UNUSED_PARAMETER(x) (void)(x) ADDED ext/sqlrr/README.txt Index: ext/sqlrr/README.txt ================================================================== --- /dev/null +++ ext/sqlrr/README.txt @@ -0,0 +1,54 @@ + +sqlrr - 10/19/2009 + +SQL Replay Recording + +SUMMARY +------------------------------------------------------- +This extension enables recording sqlite API calls that access the database so that they can be replayed or examined. + +USAGE +------------------------------------------------------ +Recording is enabled by compiling sqlite with symbolic constant SQLITE_ENABLE_SQLRR defined. +By default logs are written to /tmp/__.sqlrr, to choose another directory, set the environment variable SQLITE_REPLAY_RECORD_DIR to that path. + +FILE FORMAT +----------------------------------------------------- + file:
[]* + + header: + signature: SQLRR (5 bytes) + format-version: n (1 byte) + + sql-command: + timestamp: n (16 bytes) + type: n (1 byte) + open 0 + close 1 + exec 8 + bind-text 16 + bind-double 17 + bind-int 18 + bind-null 19 + bind-value 20 + bind-clear 21 + prep 32 + step 33 + reset 34 + finalize 35 + + open-arg-data: + close-arg-data: + exec-arg-data: + bind-text-arg-data: + bind-double-arg-data: + bind-int-arg-data: + bind-null-arg-data: + bind-value-arg-data: ??? + bind-clear-arg-data: + prep-arg-data: + step-arg-data: + reset-arg-data: + finalize-arg-data: + +NOTES ADDED ext/sqlrr/sqlrr.c Index: ext/sqlrr/sqlrr.c ================================================================== --- /dev/null +++ ext/sqlrr/sqlrr.c @@ -0,0 +1,581 @@ +/* + * sqlrr.c + */ + +#include "sqlrr.h" + +#if defined(SQLITE_ENABLE_SQLRR) + +#include +#include +#include +#include +#include +#include +#include + +#include "sqliteInt.h" +#include "vdbeInt.h" + +#define LOGSUFFIXLEN 48 + +/* + * Data types + */ +typedef struct SRRLogRef SRRLogRef; +struct SRRLogRef { + int fd; + sqlite3 *db; + const char *dbPath; + char *logPath; + int connection; + int depth; + SRRLogRef *nextRef; +}; + +/* + * Globals + */ +SRRLogRef *logRefHead = NULL; +int dbLogCount = 0; +static int srr_enabled = 1; +pthread_mutex_t srr_log_mutex; +static volatile int32_t srr_initialized = 0; + +/* + * Log management + */ +extern void SRRecInitialize() { + int go = OSAtomicCompareAndSwap32Barrier(0, 1, &srr_initialized); + if( go ){ + pthread_mutex_init(&srr_log_mutex, NULL); + } +} + +static SRRLogRef *createLog(sqlite3 *db, const char *dbPath) { + SRRLogRef *ref = NULL; + char *baseDir = getenv("SQLITE_REPLAY_RECORD_DIR"); + char logPath[MAXPATHLEN] = ""; + char suffix[LOGSUFFIXLEN] = ""; + const char *dbName = dbPath; + int len = 0; + int index = 0; + int fd = -1; + size_t out; + unsigned char version = SRR_FILE_VERSION; + + SRRecInitialize(); + + /* construct the path for the log file + * ${SQLITE_REPLAY_DIR}/__.sqlrr + */ + if (baseDir == NULL) { + baseDir = "/tmp"; /* getenv(TMPDIR) */ + } + len = strlen(baseDir); + strlcat(logPath, baseDir, MAXPATHLEN); + if ((len>0) && (baseDir[len-1] != '/')) { + strlcat(logPath, "/", MAXPATHLEN); + } + len = strlen(dbPath); + for (index = len-2; index >= 0; index --){ + if (dbPath[index] == '/') { + dbName = &dbPath[index+1]; + break; + } + } + strlcat(logPath, dbName, MAXPATHLEN); + int cNum = ++dbLogCount; + snprintf(suffix, sizeof(suffix), "_%d_%d_XXXX.sqlrr", getpid(), cNum); + len = strlcat(logPath, suffix, MAXPATHLEN); + /* make it unique if we have the space */ + if ((len + 1) < MAXPATHLEN) { + fd = mkstemps(logPath, 6); + } else { + fprintf(stderr, "Failed to create sqlite replay log path for %s [%s]\n", dbPath, logPath); + return NULL; + } + if (fd == -1) { + fprintf(stderr, "Failed to create sqlite replay log file for %s with path %s [%s]\n", dbPath, logPath, strerror(errno)); + return NULL; + } + fprintf(stdout, "Writing sqlite replay log file %s\n", logPath); + out = write(fd, SRR_FILE_SIGNATURE, SRR_FILE_SIGNATURE_LEN); + if (out!=-1) { + out = write(fd, &version, 1); + } + if (out == -1){ + fprintf(stderr, "Write failure on log [%s]: %s\n", logPath, strerror(errno)); + close(fd); + return NULL; + } + + len = strlen(logPath) + 1; + ref = (SRRLogRef *)malloc(sizeof(SRRLogRef)); + + ref->db = db; + ref->dbPath = dbPath; + ref->logPath = (char *)malloc(len * sizeof(char)); + strlcpy(ref->logPath, logPath, len); + ref->fd = fd; + ref->connection = cNum; + ref->depth = 0; + + pthread_mutex_lock(&srr_log_mutex); + ref->nextRef = logRefHead; + logRefHead = ref; + pthread_mutex_unlock(&srr_log_mutex); + return ref; +} + +static void closeLog(sqlite3 *db) { + SRRLogRef *ref = NULL; + SRRLogRef *lastRef = NULL; + + pthread_mutex_lock(&srr_log_mutex); + for (ref = logRefHead; ref != NULL; ref = ref->nextRef) { + if (ref->db == db) { + if (lastRef == NULL) { + logRefHead = ref->nextRef; + } else { + lastRef->nextRef = ref->nextRef; + } + } + } + pthread_mutex_unlock(&srr_log_mutex); + + if (ref != NULL) { + fprintf(stdout, "Closing sqlite replay log file %s\n", ref->logPath); + close(ref->fd); + free(ref->logPath); + free(ref); + } +} + +static SRRLogRef *getLog(sqlite3 *db) { + pthread_mutex_lock(&srr_log_mutex); + SRRLogRef *ref = logRefHead; + for (ref = logRefHead; ref != NULL; ref = ref->nextRef) { + if (ref->db == db) { + pthread_mutex_unlock(&srr_log_mutex); + return ref; + } + } + pthread_mutex_unlock(&srr_log_mutex); + return NULL; +} + + +/* + * SQLite recording API + */ +void SQLiteReplayRecorder(int flag) { + srr_enabled = flag; +} + +// open-arg-data: +void _SRRecOpen(sqlite3 *db, const char *path, int flags) { + if (!srr_enabled) return; + if (db) { + SRRLogRef *ref = createLog(db, path); + if (ref) { + SRRCommand code = SRROpen; + int len = strlen(path); + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out=write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { out=write(ref->fd, &(ref->connection), sizeof(ref->connection)); } + if (out!=-1) { out=write(ref->fd, &len, sizeof(len)); } + if (out!=-1) { out=write(ref->fd, path, len); } + if (out!=-1) { out=write(ref->fd, &flags, sizeof(flags)); } + if (out==-1) { + fprintf(stderr, "Error writing open to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(db); + } + } + } +} + +//close-arg-data: +void SRRecClose(sqlite3 *db) { + if (!srr_enabled) return; + if (db) { + SRRLogRef *ref = getLog(db); + if (ref) { + SRRCommand code = SRRClose; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { out = write(ref->fd, &(ref->connection), sizeof(ref->connection)); } + if (out==-1) { + fprintf(stderr, "Error writing close to log file [%s]: %s\n", ref->logPath, strerror(errno)); + } + closeLog(db); + } + } +} + +// exec-arg-data: +void SRRecExec(sqlite3 *db, const char *sql) { + if (!srr_enabled) return; + if (db) { + SRRLogRef *ref = getLog(db); + if (ref) { + if (ref->depth == 0) { + SRRCommand code = SRRExec; + int len = strlen(sql); + struct timeval tv; + size_t out; + + ref->depth = 1; + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { out = write(ref->fd, &(ref->connection), sizeof(ref->connection)); } + if (out!=-1) { out = write(ref->fd, &len, sizeof(len)); } + if (out!=-1) { out = write(ref->fd, sql, len); } + if (out==-1) { + fprintf(stderr, "Error writing exec to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(db); + } + } else { + ref->depth ++; + } + } + } +} + +void SRRecExecEnd(sqlite3 *db) { + if (!srr_enabled) return; + if (db) { + SRRLogRef *ref = getLog(db); + if (ref) { + ref->depth --; + } + } +} + +// prep-arg-data: +void _SRRecPrepare(sqlite3 *db, const char *sql, int nBytes, int saveSql, sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if ((db!=NULL)&&(pStmt!=NULL)) { + SRRLogRef *ref = getLog(db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRPrepare; + struct timeval tv; + size_t out; + int sqlLen = nBytes; + + if (sqlLen == -1) { + sqlLen = strlen(sql); + } + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { out = write(ref->fd, &(ref->connection), sizeof(ref->connection)); } + if (out!=-1) { out = write(ref->fd, &sqlLen, sizeof(sqlLen)); } + if (out!=-1) { out = write(ref->fd, sql, sqlLen); } + if (out!=-1) { out = write(ref->fd, &saveSql, sizeof(saveSql)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out==-1) { + fprintf(stderr, "Error writing prepare to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(db); + } + } + } +} + +//step-arg-data: +void SRRecStep(sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref) { + if (ref->depth == 0) { + SRRCommand code = SRRStep; + struct timeval tv; + size_t out; + + ref->depth = 1; + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out==-1) { + fprintf(stderr, "Error writing step to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } else { + ref->depth ++; + } + } + } +} + +void SRRecStepEnd(sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref) { + ref->depth --; + } + } +} + +// reset-arg-data: +void SRRecReset(sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRReset; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out==-1) { + fprintf(stderr, "Error writing reset to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// finalize-arg-data: +void SRRecFinalize(sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRFinalize; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out==-1) { + fprintf(stderr, "Error writing finalize to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-text-arg-data: +void SRRecBindText(sqlite3_stmt *pStmt, int i, const char *zData, int64_t nData) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindText; + struct timeval tv; + size_t out; + int64_t textLen = nData; + if (textLen == -1) { + textLen = strlen(zData); + } + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out!=-1) { out = write(ref->fd, &i, sizeof(i)); } + if (out!=-1) { out = write(ref->fd, &textLen, sizeof(textLen)); } + if (out!=-1) { out = write(ref->fd, zData, textLen); } + if (out==-1) { + fprintf(stderr, "Error writing bind text to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-blob-arg-data: [] +void SRRecBindBlob(sqlite3_stmt *pStmt, int i, const char *zData, int64_t nData) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindBlob; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out!=-1) { out = write(ref->fd, &i, sizeof(i)); } + if (zData == NULL) { + int64_t negNData = -nData; + if (out!=-1) { out = write(ref->fd, &negNData, sizeof(negNData)); } + } else { + if (out!=-1) { out = write(ref->fd, &nData, sizeof(nData)); } + if (out!=-1) { out = write(ref->fd, zData, nData); } + } + if (out==-1) { + fprintf(stderr, "Error writing bind blob to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-double-arg-data: +void SRRecBindDouble(sqlite3_stmt *pStmt, int i, double value) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindDouble; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out!=-1) { out = write(ref->fd, &i, sizeof(i)); } + if (out!=-1) { out = write(ref->fd, &value, sizeof(value)); } + if (out==-1) { + fprintf(stderr, "Error writing bind double to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-int-arg-data: +void SRRecBindInt64(sqlite3_stmt *pStmt, int i, int64_t value) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindInt; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out!=-1) { out = write(ref->fd, &i, sizeof(i)); } + if (out!=-1) { out = write(ref->fd, &value, sizeof(value)); } + if (out==-1) { + fprintf(stderr, "Error writing bind int to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-null-arg-data: +void SRRecBindNull(sqlite3_stmt *pStmt, int i) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindNull; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out!=-1) { out = write(ref->fd, &i, sizeof(i)); } + if (out==-1) { + fprintf(stderr, "Error writing bind null to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +// bind-value-arg-data: ??? +void SRRecBindValue(sqlite3_stmt *pStmt, int i, const sqlite3_value *value) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + fprintf(stderr, "SRRecBindValue(sqlite3_bind_value) is not yet supported, closing [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } +} + +// bind-clear-arg-data: +void SRRecClearBindings(sqlite3_stmt *pStmt) { + if (!srr_enabled) return; + if(pStmt!=NULL) { + Vdbe *v = (Vdbe *)pStmt; + SRRLogRef *ref = getLog(v->db); + if (ref && (ref->depth == 0)) { + SRRCommand code = SRRBindClear; + struct timeval tv; + size_t out; + + gettimeofday(&tv, NULL); + out = write(ref->fd, &tv, sizeof(tv)); + if (out!=-1) { out = write(ref->fd, &code, sizeof(SRRCommand)); } + if (out!=-1) { + int64_t stmtInt = (int64_t)((intptr_t)(pStmt)); + out = write(ref->fd, &stmtInt, sizeof(int64_t)); + } + if (out==-1) { + fprintf(stderr, "Error writing clear bindings to log file [%s]: %s\n", ref->logPath, strerror(errno)); + closeLog(ref->db); + } + } + } +} + +#endif /* SQLITE_ENABLE_SQLRR */ ADDED ext/sqlrr/sqlrr.h Index: ext/sqlrr/sqlrr.h ================================================================== --- /dev/null +++ ext/sqlrr/sqlrr.h @@ -0,0 +1,61 @@ +/* + * sqlrr.h + */ + +#ifndef _SQLRR_H_ +#define _SQLRR_H_ + +/* +** Header constants +*/ +#define SRR_FILE_SIGNATURE "SQLRR" +#define SRR_FILE_SIGNATURE_LEN 5 +#define SRR_FILE_VERSION 0x1 +#define SRR_FILE_VERSION_LEN 1 + +#if defined(SQLITE_ENABLE_SQLRR) + +#include "sqlite3.h" +#include + +#define SRRecOpen(A,B,C) if(!rc){_SRRecOpen(A,B,C);} +#define SRRecPrepare(A,B,C,D,E) if(!rc){_SRRecPrepare(A,B,C,D,E);} + +typedef enum { + SRROpen = 0, + SRRClose = 1, + SRRExec = 8, + SRRBindText = 16, + SRRBindBlob = 17, + SRRBindDouble = 18, + SRRBindInt = 19, + SRRBindNull = 20, + SRRBindValue = 21, + SRRBindClear = 22, + SRRPrepare = 32, + SRRStep = 33, + SRRReset = 34, + SRRFinalize = 35 +} SRRCommand; + +extern void SQLiteReplayRecorder(int flag); +extern void _SRRecOpen(sqlite3 *db, const char *path, int flags); +extern void SRRecClose(sqlite3 *db); +extern void SRRecExec(sqlite3 *db, const char *sql); +extern void SRRecExecEnd(sqlite3 *db); +extern void _SRRecPrepare(sqlite3 *db, const char *sql, int nBytes, int saveSql, sqlite3_stmt *stmt); +extern void SRRecStep(sqlite3_stmt *pStmt); +extern void SRRecStepEnd(sqlite3_stmt *pStmt); +extern void SRRecReset(sqlite3_stmt *pStmt); +extern void SRRecFinalize(sqlite3_stmt *pStmt); +extern void SRRecBindText(sqlite3_stmt *pStmt, int i, const char *zData, int64_t nData); +extern void SRRecBindBlob(sqlite3_stmt *pStmt, int i, const char *zData, int64_t nData); +extern void SRRecBindDouble(sqlite3_stmt *pStmt, int i, double value); +extern void SRRecBindInt64(sqlite3_stmt *pStmt, int i, int64_t value); +extern void SRRecBindNull(sqlite3_stmt *pStmt, int i); +extern void SRRecBindValue(sqlite3_stmt *pStmt, int i, const sqlite3_value *value); +extern void SRRecClearBindings(sqlite3_stmt *pStmt); + +#endif /* defined(SQLITE_ENABLE_SQLRR) */ + +#endif /* _SQLRR_H_ */ Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -229,10 +229,13 @@ SRC += \ $(TOP)/ext/rtree/sqlite3rtree.h \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c \ $(TOP)/ext/rtree/geopoly.c +SRC += \ + $(TOP)/ext/sqlrr/sqlrr.h \ + $(TOP)/ext/sqlrr/sqlrr.c SRC += \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/sqlite3session.h SRC += \ $(TOP)/ext/userauth/userauth.c \ @@ -476,10 +479,12 @@ EXTHDR += \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_tokenizer.h +EXTHDR += \ + $(TOP)/ext/sqlrr/sqlrr.h EXTHDR += \ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/geopoly.c EXTHDR += \ $(TOP)/ext/icu/sqliteicu.h @@ -630,14 +635,14 @@ # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # -target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c +target_source: $(SRC) $(EXTHDR) $(TOP)/tool/vdbe-compress.tcl fts5.c rm -rf tsrc mkdir tsrc - cp -f $(SRC) tsrc + cp -f $(SRC) $(EXTHDR) tsrc rm tsrc/sqlite.h.in tsrc/parse.y tclsh $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new mv vdbe.new tsrc/vdbe.c cp fts5.c fts5.h tsrc touch target_source Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -85,12 +85,18 @@ ** This routine has no effect on existing database connections. ** The shared cache setting effects only future calls to ** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). */ int sqlite3_enable_shared_cache(int enable){ +#if defined(__APPLE__) && !defined(SQLITE_TEST) && !defined(TH3_COMPATIBILITY) + /* Enable global shared cache function for debugging and unit tests, + ** but not for release */ + return SQLITE_MISUSE; +#else sqlite3GlobalConfig.sharedCacheEnabled = enable; return SQLITE_OK; +#endif } #endif Index: src/btreeInt.h ================================================================== --- src/btreeInt.h +++ src/btreeInt.h @@ -231,26 +231,10 @@ /* Forward declarations */ typedef struct MemPage MemPage; typedef struct BtLock BtLock; typedef struct CellInfo CellInfo; -/* -** This is a magic string that appears at the beginning of every -** SQLite database in order to identify the file as a real database. -** -** You can change this value at compile-time by specifying a -** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The -** header must be exactly 16 bytes including the zero-terminator so -** the string itself should be 15 characters long. If you change -** the header, then your custom library will not be able to read -** databases generated by the standard tools and the standard tools -** will not be able to read databases created by your custom library. -*/ -#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ -# define SQLITE_FILE_HEADER "SQLite format 3" -#endif - /* ** Page type flags. An ORed combination of these flags appear as the ** first byte of on-disk image of every BTree page. */ #define PTF_INTKEY 0x01 Index: src/legacy.c ================================================================== --- src/legacy.c +++ src/legacy.c @@ -14,10 +14,13 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. */ #include "sqliteInt.h" +#ifdef SQLITE_ENABLE_SQLRR +# include "sqlrr.h" +#endif /* ** Execute SQL code. Return one of the SQLITE_ success/failure ** codes. Also write an error message into memory obtained from ** malloc() and make *pzErrMsg point to that message. @@ -40,11 +43,13 @@ char **azCols = 0; /* Names of result columns */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; if( zSql==0 ) zSql = ""; - +#ifdef SQLITE_ENABLE_SQLRR + SRRecExec(db, zSql); +#endif sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ int nCol = 0; char **azVals = 0; @@ -121,10 +126,13 @@ } exec_out: if( pStmt ) sqlite3VdbeFinalize((Vdbe *)pStmt); sqlite3DbFree(db, azCols); +#ifdef SQLITE_ENABLE_SQLRR + SRRecExecEnd(db); +#endif rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3DbStrDup(0, sqlite3_errmsg(db)); if( *pzErrMsg==0 ){ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -14,10 +14,13 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. */ #include "sqliteInt.h" +#ifdef SQLITE_ENABLE_SQLRR +# include "sqlrr.h" +#endif #ifdef SQLITE_ENABLE_FTS3 # include "fts3.h" #endif #ifdef SQLITE_ENABLE_RTREE # include "rtree.h" @@ -1306,10 +1309,14 @@ sqlite3_mutex_free(db->mutex); assert( sqlite3LookasideUsed(db,0)==0 ); if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } +#ifdef SQLITE_ENABLE_SQLRR + SRRecClose(db); +#endif + sqlite3_free(db); } /* ** Rollback all database files. If tripCode is not SQLITE_OK, then @@ -1418,10 +1425,11 @@ case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_IOERR_VNODE: zName = "SQLITE_IOERR_VNODE"; break; case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; case SQLITE_FULL: zName = "SQLITE_FULL"; break; case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; @@ -2747,10 +2755,48 @@ } db->aLimit[limitId] = newLimit; } return oldLimit; /* IMP: R-53341-35419 */ } +#if defined(SQLITE_ENABLE_AUTO_PROFILE) +/* stderr logging */ +void _sqlite_auto_profile(void *aux, const char *sql, u64 ns); +void _sqlite_auto_trace(void *aux, const char *sql); +void _sqlite_auto_profile(void *aux, const char *sql, u64 ns) { +#pragma unused(aux) + fprintf(stderr, "Query: %s\n Execution Time: %llu ms\n", sql, ns / 1000000); +} +void _sqlite_auto_trace(void *aux, const char *sql) { + fprintf(stderr, "TraceSQL(%p): %s\n", aux, sql); +} + +/* syslog logging */ +#include +static aslclient autolog_client = NULL; +static void _close_asl_log() { + if( NULL!=autolog_client ){ + asl_close(autolog_client); + autolog_client = NULL; + } +} +static void _open_asl_log() { + if( NULL==autolog_client ){ + autolog_client = asl_open("SQLite", NULL, 0); + atexit(_close_asl_log); + } +} + +void _sqlite_auto_profile_syslog(void *aux, const char *sql, u64 ns); +void _sqlite_auto_trace_syslog(void *aux, const char *sql); +void _sqlite_auto_profile_syslog(void *aux, const char *sql, u64 ns) { +#pragma unused(aux) + asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "Query: %s\n Execution Time: %llu ms\n", sql, ns / 1000000); +} +void _sqlite_auto_trace_syslog(void *aux, const char *sql) { + asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "TraceSQL(%p): %s\n", aux, sql); +} +#endif /* ** This function is used to parse both URIs and non-URI filenames passed by the ** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database ** URIs specified as part of ATTACH statements. @@ -3007,10 +3053,62 @@ *pFlags = flags; *pzFile = zFile; return rc; } +#if defined(SQLITE_ENABLE_AUTO_PROFILE) +#define SQLITE_AUTOLOGGING_STDERR 1 +#define SQLITE_AUTOLOGGING_SYSLOG 2 +static void enableAutoLogging( + sqlite3 *db +){ + char *envprofile = getenv("SQLITE_AUTO_PROFILE"); + + if( envprofile!=NULL ){ + int where = 0; + if( !strncasecmp("1", envprofile, 1) ){ + if( isatty(2) ){ + where = SQLITE_AUTOLOGGING_STDERR; + }else{ + where = SQLITE_AUTOLOGGING_SYSLOG; + } + } else if( !strncasecmp("stderr", envprofile, 6) ){ + where = SQLITE_AUTOLOGGING_STDERR; + } else if( !strncasecmp("syslog", envprofile, 6) ){ + where = SQLITE_AUTOLOGGING_SYSLOG; + } + if( where==SQLITE_AUTOLOGGING_STDERR ){ + sqlite3_profile(db, _sqlite_auto_profile, db); + }else if( where==SQLITE_AUTOLOGGING_SYSLOG ){ + _open_asl_log(); + sqlite3_profile(db, _sqlite_auto_profile_syslog, db); + } + } + char *envtrace = getenv("SQLITE_AUTO_TRACE"); + if( envtrace!=NULL ){ + int where = 0; + if( !strncasecmp("1", envtrace, 1) ){ + if( isatty(2) ){ + where = SQLITE_AUTOLOGGING_STDERR; + }else{ + where = SQLITE_AUTOLOGGING_SYSLOG; + } + } else if( !strncasecmp("stderr", envtrace, 6) ){ + where = SQLITE_AUTOLOGGING_STDERR; + } else if( !strncasecmp("syslog", envtrace, 6) ){ + where = SQLITE_AUTOLOGGING_SYSLOG; + } + if( where==SQLITE_AUTOLOGGING_STDERR ){ + sqlite3_trace(db, _sqlite_auto_trace, db); + }else if( where==SQLITE_AUTOLOGGING_SYSLOG ){ + _open_asl_log(); + sqlite3_trace(db, _sqlite_auto_trace_syslog, db); + } + } +} +#endif + /* ** This routine does the core work of extracting URI parameters from a ** database filename for the sqlite3_uri_parameter() interface. */ static const char *uriParameter(const char *zFilename, const char *zParam){ @@ -3021,12 +3119,10 @@ if( x==0 ) return zFilename; zFilename += sqlite3Strlen30(zFilename) + 1; } return 0; } - - /* ** This routine does the work of opening a database on behalf of ** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" ** is UTF-8 encoded. @@ -3395,11 +3491,34 @@ sqlite3_close(db); db = 0; }else if( rc!=SQLITE_OK ){ db->magic = SQLITE_MAGIC_SICK; } +#if defined(__APPLE__) && ENABLE_FORCE_WAL + if( db && !rc ){ + if ((0 == access("/var/db/enableForceWAL", R_OK))) { +#ifdef SQLITE_DEBUG + fprintf(stderr, "SQLite WAL journal_mode ENABLED by default.\n"); +#endif + + sqlite3_exec(db, "pragma journal_mode=wal", NULL, NULL, NULL); +#ifdef SQLITE_DEBUG +// } else { +// fprintf(stderr, "SQLite WAL journal_mode NOT ENABLED by default.\n"); +#endif + } + } +#endif +#if defined(SQLITE_ENABLE_AUTO_PROFILE) + if( db && !rc ){ + enableAutoLogging(db); + } +#endif *ppDb = db; +#ifdef SQLITE_ENABLE_SQLRR + SRRecOpen(db, zFilename, flags); +#endif #ifdef SQLITE_ENABLE_SQLLOG if( sqlite3GlobalConfig.xSqllog ){ /* Opening a db handle. Fourth parameter is passed 0. */ void *pArg = sqlite3GlobalConfig.pSqllogArg; sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); @@ -3838,18 +3957,27 @@ *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); rc = SQLITE_OK; }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); rc = SQLITE_OK; +#ifndef SQLITE_OMIT_WAL + if( (rc==SQLITE_OK)&&(op==SQLITE_FCNTL_LAST_ERRNO)&&(*(int *)pArg==0) ){ + sqlite3_file *pWalFd = sqlite3PagerWalFile(pPager); + if( pWalFd&&(pWalFd->pMethods) ){ + rc = sqlite3OsFileControl(pWalFd, op, pArg); + } + } +#endif }else if( op==SQLITE_FCNTL_DATA_VERSION ){ *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); rc = SQLITE_OK; }else{ rc = sqlite3OsFileControl(fd, op, pArg); } sqlite3BtreeLeave(pBtree); } + sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } /* @@ -4443,10 +4571,36 @@ } #endif pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + +#include "sqlite3_private.h" + +/* +** Testing a file path for sqlite locks held by a process ID. +** Returns SQLITE_LOCKSTATE_ON if locks are present on path +** that would prevent writing to the database. +*/ +int _sqlite3_lockstate(const char *path, pid_t pid){ + sqlite3 *db = NULL; + + if( sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK ){ + LockstatePID lockstate = {pid, -1}; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_LOCKSTATE_PID, &lockstate); + sqlite3_close(db); + int state = lockstate.state; + return state; + } + if( NULL!=db ){ + sqlite3_close(db); /* need to close even if open returns an error */ + } + return SQLITE_LOCKSTATE_ERROR; +} + +#endif /* SQLITE_ENABLE_APPLE_SPI */ #ifdef SQLITE_ENABLE_SNAPSHOT /* ** Obtain a snapshot handle for the snapshot of database zDb currently ** being read by handle db. Index: src/os.c ================================================================== --- src/os.c +++ src/os.c @@ -208,16 +208,22 @@ sqlite3_file *pFile, int flags, int *pFlagsOut ){ int rc; + int openFlags; DO_OS_MALLOC_TEST(0); /* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); +#if SQLITE_ENABLE_DATA_PROTECTION + openFlags = flags & (0x1087f7f | SQLITE_OPEN_FILEPROTECTION_MASK); +#else + openFlags = flags & 0x1087f7f; +#endif + rc = pVfs->xOpen(pVfs, zPath, pFile, openFlags, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ DO_OS_MALLOC_TEST(0); Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -99,10 +99,11 @@ # include #endif #if SQLITE_ENABLE_LOCKING_STYLE # include +# include # include # include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ /* @@ -235,13 +236,16 @@ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ #endif int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) int openFlags; /* The flags specified at open() */ #endif +#if SQLITE_ENABLE_DATA_PROTECTION + int protFlags; /* Data protection flags from unixOpen */ +#endif #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) unsigned fsFlags; /* cached details from statfs() */ #endif #ifdef SQLITE_ENABLE_SETLK_TIMEOUT unsigned iBusyTimeout; /* Wait this many millisec on locks */ @@ -322,10 +326,294 @@ #if SQLITE_THREADSAFE #define threadid pthread_self() #else #define threadid 0 #endif + +#ifdef __APPLE__ +#define SQLITE_ENABLE_SUPERLOCK 1 +#endif + +#if SQLITE_ENABLE_SUPERLOCK +#include "sqlite3.h" +#include +#include + +/* +** A structure to collect a busy-handler callback and argument and a count +** of the number of times it has been invoked. +*/ +struct SuperlockBusy { + int (*xBusy)(void*,int); /* Pointer to busy-handler function */ + void *pBusyArg; /* First arg to pass to xBusy */ + int nBusy; /* Number of times xBusy has been invoked */ +}; +typedef struct SuperlockBusy SuperlockBusy; + +/* +** An instance of the following structure is allocated for each active +** superlock. The opaque handle returned by sqlite3demo_superlock() is +** actually a pointer to an instance of this structure. +*/ +struct Superlock { + sqlite3 *db; /* Database handle used to lock db */ + int bWal; /* True if db is a WAL database */ +}; +typedef struct Superlock Superlock; + +/* +** The pCtx pointer passed to this function is actually a pointer to a +** SuperlockBusy structure. Invoke the busy-handler function encapsulated +** by the structure and return the result. +*/ +static int superlockBusyHandler(void *pCtx, int UNUSED){ + SuperlockBusy *pBusy = (SuperlockBusy *)pCtx; + if( pBusy->xBusy==0 ) return 0; + return pBusy->xBusy(pBusy->pBusyArg, pBusy->nBusy++); +} + +/* +** This function is used to determine if the main database file for +** connection db is open in WAL mode or not. If no error occurs and the +** database file is in WAL mode, set *pbWal to true and return SQLITE_OK. +** If it is not in WAL mode, set *pbWal to false. +** +** If an error occurs, return an SQLite error code. The value of *pbWal +** is undefined in this case. +*/ +static int superlockIsWal(Superlock *pLock){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt; /* Compiled PRAGMA journal_mode statement */ + + rc = sqlite3_prepare(pLock->db, "PRAGMA main.journal_mode", -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + + pLock->bWal = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zMode = (const char *)sqlite3_column_text(pStmt, 0); + if( zMode && strlen(zMode)==3 && sqlite3_strnicmp("wal", zMode, 3)==0 ){ + pLock->bWal = 1; + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** Obtain an exclusive shm-lock on nByte bytes starting at offset idx +** of the file fd. If the lock cannot be obtained immediately, invoke +** the busy-handler until either it is obtained or the busy-handler +** callback returns 0. +*/ +static int superlockShmLock( + sqlite3_file *fd, /* Database file handle */ + int idx, /* Offset of shm-lock to obtain */ + int nByte, /* Number of consective bytes to lock */ + SuperlockBusy *pBusy /* Busy-handler wrapper object */ +){ + int rc; + int (*xShmLock)(sqlite3_file*, int, int, int) = fd->pMethods->xShmLock; + do { + rc = xShmLock(fd, idx, nByte, SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE); + }while( rc==SQLITE_BUSY && superlockBusyHandler((void *)pBusy, 0) ); + return rc; +} + +/* +** Obtain the extra locks on the database file required for WAL databases. +** Invoke the supplied busy-handler as required. +*/ +static int superlockWalLock( + sqlite3 *db, /* Database handle open on WAL database */ + SuperlockBusy *pBusy /* Busy handler wrapper object */ +){ + int rc; /* Return code */ + sqlite3_file *fd = 0; /* Main database file handle */ + void volatile *p = 0; /* Pointer to first page of shared memory */ + + /* Obtain a pointer to the sqlite3_file object open on the main db file. */ + rc = sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&fd); + if( rc!=SQLITE_OK ) return rc; + + /* Obtain the "recovery" lock. Normally, this lock is only obtained by + ** clients running database recovery. + */ + rc = superlockShmLock(fd, 2, 1, pBusy); + if( rc!=SQLITE_OK ) return rc; + + /* Zero the start of the first shared-memory page. This means that any + ** clients that open read or write transactions from this point on will + ** have to run recovery before proceeding. Since they need the "recovery" + ** lock that this process is holding to do that, no new read or write + ** transactions may now be opened. Nor can a checkpoint be run, for the + ** same reason. + */ + rc = fd->pMethods->xShmMap(fd, 0, 32*1024, 1, &p); + if( rc!=SQLITE_OK ) return rc; + memset((void *)p, 0, 32); + + /* Obtain exclusive locks on all the "read-lock" slots. Once these locks + ** are held, it is guaranteed that there are no active reader, writer or + ** checkpointer clients. + */ + rc = superlockShmLock(fd, 3, SQLITE_SHM_NLOCK-3, pBusy); + return rc; +} + +/* +** Release a superlock held on a database file. The argument passed to +** this function must have been obtained from a successful call to +** sqlite3demo_superlock(). +*/ +static void sqlite3demo_superunlock(void *pLock){ + Superlock *p = (Superlock *)pLock; + if( p->bWal ){ + int rc; /* Return code */ + int flags = SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE; + sqlite3_file *fd = 0; + rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_FILE_POINTER, + (void *)&fd); + if( rc==SQLITE_OK ){ + fd->pMethods->xShmLock(fd, 2, 1, flags); + fd->pMethods->xShmLock(fd, 3, SQLITE_SHM_NLOCK-3, flags); + } + } + sqlite3_close(p->db); + sqlite3_free(p); +} + +/* +** Obtain a superlock on the database file identified by zPath, using the +** locking primitives provided by VFS zVfs. If successful, SQLITE_OK is +** returned and output variable *ppLock is populated with an opaque handle +** that may be used with sqlite3demo_superunlock() to release the lock. +** +** If an error occurs, *ppLock is set to 0 and an SQLite error code +** (e.g. SQLITE_BUSY) is returned. +** +** If a required lock cannot be obtained immediately and the xBusy parameter +** to this function is not NULL, then xBusy is invoked in the same way +** as a busy-handler registered with SQLite (using sqlite3_busy_handler()) +** until either the lock can be obtained or the busy-handler function returns +** 0 (indicating "give up"). +*/ +static int sqlite3demo_superlock( + const char *zPath, /* Path to database file to lock */ + const char *zVfs, /* VFS to use to access database file */ + int flags, /* Additional flags to pass to sqlite3_open_v2 */ + int (*xBusy)(void*,int), /* Busy handler callback */ + void *pBusyArg, /* Context arg for busy handler */ + void **ppLock /* OUT: Context to pass to superunlock() */ +){ + SuperlockBusy busy = {0, 0, 0}; /* Busy handler wrapper object */ + int rc; /* Return code */ + Superlock *pLock; + + pLock = sqlite3_malloc(sizeof(Superlock)); + if( !pLock ) return SQLITE_NOMEM; + memset(pLock, 0, sizeof(Superlock)); + + /* Open a database handle on the file to superlock. */ + rc = sqlite3_open_v2( + zPath, &pLock->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|flags, zVfs + ); + + /* Install a busy-handler and execute a BEGIN EXCLUSIVE. If this is not + ** a WAL database, this is all we need to do. + ** + ** A wrapper function is used to invoke the busy-handler instead of + ** registering the busy-handler function supplied by the user directly + ** with SQLite. This is because the same busy-handler function may be + ** invoked directly later on when attempting to obtain the extra locks + ** required in WAL mode. By using the wrapper, we are able to guarantee + ** that the "nBusy" integer parameter passed to the users busy-handler + ** represents the total number of busy-handler invocations made within + ** this call to sqlite3demo_superlock(), including any made during the + ** "BEGIN EXCLUSIVE". + */ + if( rc==SQLITE_OK ){ + busy.xBusy = xBusy; + busy.pBusyArg = pBusyArg; + sqlite3_busy_handler(pLock->db, superlockBusyHandler, (void *)&busy); + rc = sqlite3_exec(pLock->db, "BEGIN EXCLUSIVE", 0, 0, 0); + } + + /* If the BEGIN EXCLUSIVE was executed successfully and this is a WAL + ** database, call superlockWalLock() to obtain the extra locks required + ** to prevent readers, writers and/or checkpointers from accessing the + ** db while this process is holding the superlock. + ** + ** Before attempting any WAL locks, commit the transaction started above + ** to drop the WAL read and write locks currently held. Otherwise, the + ** new WAL locks may conflict with the old. + */ + if( rc==SQLITE_OK ){ + if( SQLITE_OK==(rc = superlockIsWal(pLock)) && pLock->bWal ){ + rc = sqlite3_exec(pLock->db, "COMMIT", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = superlockWalLock(pLock->db, &busy); + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3demo_superunlock(pLock); + *ppLock = 0; + }else{ + *ppLock = pLock; + } + + return rc; +} + +/* A corrupt DB won't work with the sql-based locking attempt, grab an +** exclusive lock and return SQLITE_OK or SQLITE_BUSY if the lock fails +** returns the current lock level held on sqlite3_file +*/ +static int sqlite3demo_superlock_corrupt( + sqlite3_file *id, + int eTargetFileLock, + int *pFileLock +){ + unixFile *pFile = (unixFile*)id; + int eFileLock = pFile->eFileLock; + int rc = SQLITE_OK; + + if( eFileLockpMethod->xLock(id, SQLITE_LOCK_SHARED); + } + if( !rc && SQLITE_LOCK_SHAREDpMethod->xLock(id, SQLITE_LOCK_EXCLUSIVE); + } + if( rc ){ + if( pFile->eFileLock > eFileLock ){ + pFile->pMethod->xUnlock(id, eFileLock); + } + return rc; + } + if (pFileLock) { + *pFileLock = eFileLock; + } + return SQLITE_OK; +} + +static int sqlite3demo_superunlock_corrupt(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + + if( pFile->eFileLock > eFileLock ){ + rc = pFile->pMethod->xUnlock(id, SQLITE_LOCK_SHARED); + } + if( pFile->eFileLock > eFileLock ){ + int unlockRC = pFile->pMethod->xUnlock(id, SQLITE_LOCK_NONE); + if (!rc) rc = unlockRC; + } + return rc; +} + +#endif /* SQLITE_ENABLE_SUPERLOCK */ + /* ** HAVE_MREMAP defaults to true on Linux and false everywhere else. */ #if !defined(HAVE_MREMAP) @@ -1306,12 +1594,23 @@ UnixUnusedFd *p; UnixUnusedFd *pNext; assert( unixFileMutexHeld(pFile) ); for(p=pInode->pUnused; p; p=pNext){ pNext = p->pNext; +#if OSCLOSE_CHECK_CLOSE_IOERR + if( close(p->fd) ){ + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR_CLOSE; + p->pNext = pError; + pError = p; + }else{ + sqlite3_free(p); + } +#else robust_close(pFile, p->fd, __LINE__); sqlite3_free(p); +#endif } pInode->pUnused = 0; } /* @@ -1527,12 +1826,18 @@ lock.l_whence = SEEK_SET; lock.l_start = RESERVED_BYTE; lock.l_len = 1; lock.l_type = F_WRLCK; if( osFcntl(pFile->h, F_GETLK, &lock) ){ +#if OSLOCKING_CHECK_BUSY_IOERR + int tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + storeLastErrno(pFile, tErrno); +#else rc = SQLITE_IOERR_CHECKRESERVEDLOCK; storeLastErrno(pFile, errno); +#endif } else if( lock.l_type!=F_UNLCK ){ reserved = 1; } } #endif @@ -1598,11 +1903,11 @@ ** and is read-only. ** ** Zero is returned if the call completes successfully, or -1 if a call ** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). */ -static int unixFileLock(unixFile *pFile, struct flock *pLock){ +static int unixFileLock(unixFile *pFile, struct flock *pLock, int nRetry){ int rc; unixInodeInfo *pInode = pFile->pInode; assert( pInode!=0 ); assert( sqlite3_mutex_held(pInode->pLockMutex) ); if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ @@ -1761,14 +2066,14 @@ if( eFileLock==SHARED_LOCK || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockeFileLock = SHARED_LOCK; @@ -1829,11 +2138,11 @@ }else{ lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; } - if( unixFileLock(pFile, &lock) ){ + if( unixFileLock(pFile, &lock, 0) ){ tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( rc!=SQLITE_BUSY ){ storeLastErrno(pFile, tErrno); } @@ -1954,35 +2263,50 @@ lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock(pFile, &lock, 10)==(-1) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); +#else rc = SQLITE_IOERR_UNLOCK; +#endif storeLastErrno(pFile, tErrno); goto end_unlock; } lock.l_type = F_RDLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock(pFile, &lock, 10)==(-1) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); +#else + rc = SQLITE_IOERR_UNLOCK; +#endif if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } goto end_unlock; } lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST+divSize; lock.l_len = SHARED_SIZE-divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock(pFile, &lock, 10)==(-1) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); +#else rc = SQLITE_IOERR_UNLOCK; +#endif + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } storeLastErrno(pFile, tErrno); goto end_unlock; } }else #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ @@ -1989,32 +2313,48 @@ { lock.l_type = F_RDLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; - if( unixFileLock(pFile, &lock) ){ + if( unixFileLock(pFile, &lock, 10) ){ + int tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); +#else /* In theory, the call to unixFileLock() cannot fail because another ** process is holding an incompatible lock. If it does, this ** indicates that the other process is not following the locking ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning ** SQLITE_BUSY would confuse the upper layer (in practice it causes ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; - storeLastErrno(pFile, errno); + storeLastErrno(pFile, tErrno); +#endif + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } goto end_unlock; } } } lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = PENDING_BYTE; lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock(pFile, &lock, 10)==0 ){ pInode->eFileLock = SHARED_LOCK; }else{ +#if OSLOCKING_CHECK_BUSY_IOERR + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } +#else rc = SQLITE_IOERR_UNLOCK; storeLastErrno(pFile, errno); +#endif goto end_unlock; } } if( eFileLock==NO_LOCK ){ /* Decrement the shared lock counter. Release the lock using an @@ -2024,15 +2364,23 @@ pInode->nShared--; if( pInode->nShared==0 ){ lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = lock.l_len = 0L; - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock(pFile, &lock, 10)==0 ){ pInode->eFileLock = NO_LOCK; }else{ +#if OSLOCKING_CHECK_BUSY_IOERR + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } +#else rc = SQLITE_IOERR_UNLOCK; storeLastErrno(pFile, errno); +#endif pInode->eFileLock = NO_LOCK; pFile->eFileLock = NO_LOCK; } } @@ -2184,11 +2532,28 @@ /* ** Close the file. */ static int nolockClose(sqlite3_file *id) { - return closeUnixFile(id); + int rc = SQLITE_OK; + unixFile *pFile = (unixFile *)id; + unixEnterMutex(); + if( pFile->pInode ){ + assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); + if( pFile->pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->pUnused list. It will be automatically closed + ** when the last lock is cleared. + */ + setPendingFd(pFile); + } + releaseInodeInfo(pFile); + } + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; } /******************* End of the no-op lock implementation ********************* ******************************************************************************/ @@ -2350,11 +2715,15 @@ if( rc<0 ){ int tErrno = errno; if( tErrno==ENOENT ){ rc = SQLITE_OK; }else{ +#if OSLOCKING_CHECK_BUSY_IOERR + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); +#else rc = SQLITE_IOERR_UNLOCK; +#endif storeLastErrno(pFile, tErrno); } return rc; } pFile->eFileLock = NO_LOCK; @@ -2432,11 +2801,19 @@ /* got the lock, unlock it */ lrc = robust_flock(pFile->h, LOCK_UN); if ( lrc ) { int tErrno = errno; /* unlock failed with an error */ +#if OSLOCKING_CHECK_BUSY_IOERR + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); +#else lrc = SQLITE_IOERR_UNLOCK; +#endif + if( IS_LOCK_ERROR(lrc) ){ + storeLastErrno(pFile, tErrno); + rc = lrc; + } storeLastErrno(pFile, tErrno); rc = lrc; } } else { int tErrno = errno; @@ -3549,10 +3926,16 @@ # define HAVE_FULLFSYNC 1 #else # define HAVE_FULLFSYNC 0 #endif +#ifdef SQLITE_USE_REQUEST_FULLFSYNC +#import +#import +static OSSpinLock notify_lock = 0; +#define REQUEST_FULLSYNC_NOTIFICATION "com.apple.reqsync" +#endif /* ** The fsync() system call does not work as advertised on many ** unix systems. The following procedure is an attempt to make ** it work better. @@ -3613,11 +3996,20 @@ struct stat buf; rc = osFstat(fd, &buf); } #elif HAVE_FULLFSYNC if( fullSync ){ +#ifdef SQLITE_USE_REQUEST_FULLFSYNC + rc = osFsync(fd); + if (!rc) { + OSSpinLockLock(¬ify_lock); + rc = notify_post(REQUEST_FULLSYNC_NOTIFICATION); + OSSpinLockUnlock(¬ify_lock); + } +#else rc = osFcntl(fd, F_FULLFSYNC, 0); +#endif }else{ rc = 1; } /* If the FULLFSYNC failed, fall back to attempting an fsync(). ** It shouldn't be possible for fullfsync to fail on the local @@ -3744,16 +4136,24 @@ OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, HAVE_FULLFSYNC, isFullsync)); rc = osOpenDirectory(pFile->zPath, &dirfd); if( rc==SQLITE_OK ){ full_fsync(dirfd, 0, 0); +#if OSCLOSE_CHECK_CLOSE_IOERR + if( close(pFile->dirfd) ){ + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR_DIR_CLOSE; + } +#else robust_close(pFile, dirfd, __LINE__); +#endif }else{ assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; + } return rc; } /* @@ -3851,10 +4251,11 @@ if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ if( osFstat(pFile->h, &buf) ){ + storeLastErrno(pFile, errno); return SQLITE_IOERR_FSTAT; } nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; if( nSize>(i64)buf.st_size ){ @@ -3907,10 +4308,312 @@ } #endif return SQLITE_OK; } + + +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +#include "sqlite3_private.h" +#include +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath); +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE +static int isProxyLockingMode(unixFile *); +#endif + +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static int unixTruncateDatabase(unixFile *, int); + +static int unixInvalidateSupportFiles(unixFile *, int); + +static int findCreateFileMode(const char *, int, mode_t*, uid_t *,gid_t *); + +/* opens a read/write connection to a file zName inheriting the appropriate +** user/perms from the database file if running as root. Returns the file +** descriptor by reference +*/ +static int unixOpenChildFile( + const char *zName, + int openFlags, + int dbOpenFlags, + int protFlags, + int *pFd +){ + int fd = -1; + mode_t openMode; /* Permissions to create file with */ + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + int rc; + + assert(pFd!=NULL); + rc = findCreateFileMode(zName, dbOpenFlags, &openMode, &uid, &gid); + if( rc!=SQLITE_OK ){ + return rc; + } + fd = robust_open(zName, openFlags, openMode); + OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); + if( fd<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + return rc; + } + /* if we're opening the wal or journal and running as root, set + ** the journal uid/gid */ + if( dbOpenFlags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + uid_t euid = geteuid(); + if( euid==0 && (euid!=uid || getegid()!=gid) ){ + if( fchown(fd, uid, gid) ){ + rc = SQLITE_CANTOPEN_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + *pFd = fd; + } else { + *pFd = -1; + close(fd); + } + return rc; +} + +static int unixReplaceDatabase(unixFile *pFile, sqlite3 *srcdb) { + sqlite3_file *id = (sqlite3_file *)pFile; + Btree *pSrcBtree = NULL; + sqlite3_file *src_file = NULL; + unixFile *pSrcFile = NULL; + char srcWalPath[MAXPATHLEN+5]; + int srcWalFD = -1; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + sqlite3 *srcdb2 = NULL; + copyfile_state_t s; + int corruptSrcFileLock = 0; + int corruptDstFileLock = 0; + int isSrcCorrupt = 0; + int isDstCorrupt = 0; + + if( !sqlite3SafetyCheckOk(srcdb) ){ + return SQLITE_MISUSE; + } + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isDstCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, + &corruptDstFileLock); + } + if( rc ){ + return rc; + } + } + /* get the src file descriptor adhering to the db struct access rules + ** this code is modeled after sqlite3_file_control() in main.c + */ + sqlite3_mutex_enter(srcdb->mutex); + if( srcdb->nDb>0 ){ + pSrcBtree = srcdb->aDb[0].pBt; + } + if( pSrcBtree ){ + Pager *pSrcPager; + sqlite3BtreeEnter(pSrcBtree); + pSrcPager = sqlite3BtreePager(pSrcBtree); + assert( pSrcPager!=0 ); + src_file = sqlite3PagerFile(pSrcPager); + assert( src_file!=0 ); + if( src_file->pMethods ){ + int srcFlags = 0; + pSrcFile = (unixFile *)src_file; +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { + srcFlags = SQLITE_OPEN_READWRITE; + } else { + srcFlags = SQLITE_OPEN_READONLY; + } +#else + srcFlags = SQLITE_OPEN_READWRITE; +#endif +#if SQLITE_ENABLE_DATA_PROTECTION + srcFlags |= pSrcFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pSrcFile) ){ + srcFlags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); + if( rc==SQLITE_OK ){ + /* start a deferred transaction and read to establish a read lock */ + rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", + 0, 0, 0); + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isSrcCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, + &corruptSrcFileLock); + } + } + } + } + if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ + rc = SQLITE_INTERNAL; + } + if( rc!=SQLITE_OK ){ + goto end_replace_database; + } + /* both databases are locked appropriately, copy the src wal journal if + ** one exists and then the actual database file + */ + strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); + strlcat(srcWalPath, "-wal", MAXPATHLEN+5); + srcWalFD = open(srcWalPath, O_RDONLY); + if( !(srcWalFD<0) ){ + char dstWalPath[MAXPATHLEN+5]; + int dstWalFD = -1; + int protFlags = 0; + strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); + strlcat(dstWalPath, "-wal", MAXPATHLEN+5); + + rc = unixOpenChildFile(dstWalPath, O_RDWR|O_CREAT, SQLITE_OPEN_WAL, + protFlags, &dstWalFD); + if( rc==SQLITE_OK ){ + s = copyfile_state_alloc(); + lseek(srcWalFD, 0, SEEK_SET); + lseek(dstWalFD, 0, SEEK_SET); + if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_DATA) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + storeLastErrno(pFile, err); + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + close(dstWalFD); + } + close(srcWalFD); + } + if( rc==SQLITE_OK ){ + /* before we copy, ensure that the file change counter will be modified */ + uint32_t srcChange = 0; + uint32_t dstChange = 0; + pread(pSrcFile->h, &srcChange, 4, 24); + pread(pFile->h, &dstChange, 4, 24); + + /* copy the actual database */ + s = copyfile_state_alloc(); + lseek(pSrcFile->h, 0, SEEK_SET); + lseek(pFile->h, 0, SEEK_SET); + if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_DATA) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + storeLastErrno(pFile, err); + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + + if (srcChange == dstChange) { + /* modify the change counter to force page zero to be reloaded */ + dstChange ++; + pwrite(pFile->h, &dstChange, 4, 24); + } + } + if( isSrcCorrupt ){ + sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); + }else{ + /* done with the source db so end the transaction */ + sqlite3_exec(srcdb2, "COMMIT", 0, 0, 0); + } + /* zero out any old journal clutter */ + if( rc==SQLITE_OK ){ + int skipWAL = (srcWalFD<0)?0:1; + unixInvalidateSupportFiles(pFile, skipWAL); + } + +end_replace_database: + if( pSrcBtree ){ + sqlite3_close(srcdb2); + sqlite3BtreeLeave(pSrcBtree); + } + sqlite3_mutex_leave(srcdb->mutex); + if( isDstCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); + }else{ + sqlite3demo_superunlock(pLock); + } + return rc; +} +#define SQLITE_FILE_HEADER_LEN 16 +/* Check for a conflicting lock. If one is found, print an this + ** on standard output using the format string given and return 1. + ** If there are no conflicting locks, return 0. + */ +static int unixIsLocked( + pid_t pid, /* PID to test for lock owner */ + int h, /* File descriptor to check */ + int type, /* F_RDLCK or F_WRLCK */ + unsigned int iOfst, /* First byte of the lock */ + unsigned int iCnt, /* Number of bytes in the lock range */ + const char *zType /* Type of lock */ +){ + struct flock lk; + int err; + + memset(&lk, 0, sizeof(lk)); + lk.l_type = type; + lk.l_whence = SEEK_SET; + lk.l_start = iOfst; + lk.l_len = iCnt; + + if( pid!=SQLITE_LOCKSTATE_ANYPID ){ +#ifndef F_GETLKPID +# warning F_GETLKPID undefined, _sqlite3_lockstate falling back to F_GETLK + err = fcntl(h, F_GETLK, &lk); +#else + lk.l_pid = pid; + err = fcntl(h, F_GETLKPID, &lk); +#endif + }else{ + err = fcntl(h, F_GETLK, &lk); + } + + if( err==(-1) ){ + fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); + return -1; + } + + if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ +#ifdef SQLITE_DEBUG + fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); +#endif + return 1; + } + return 0; +} + +static int unixLockstatePid(unixFile *, pid_t, int *); + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + /* ** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** @@ -4042,10 +4745,30 @@ case SQLITE_FCNTL_SET_LOCKPROXYFILE: case SQLITE_FCNTL_GET_LOCKPROXYFILE: { return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + case SQLITE_FCNTL_TRUNCATE_DATABASE: { + return unixTruncateDatabase(pFile, (pArg ? (*(int *)pArg) : 0)); + } + case SQLITE_FCNTL_REPLACE_DATABASE: { + return unixReplaceDatabase(pFile, (sqlite3 *)pArg); + } + case SQLITE_FCNTL_LOCKSTATE_PID: { + LockstatePID *pLockstate; + int rc; + + if( pArg==NULL ){ + return SQLITE_MISUSE; + } + pLockstate = (LockstatePID *)pArg; + rc = unixLockstatePid(pFile, pLockstate->pid, &(pLockstate->state)); + return rc; + } + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ } return SQLITE_NOTFOUND; } /* @@ -4472,10 +5195,14 @@ rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); } return rc; } +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static const char *proxySharedMemoryBasePath(unixFile *); +#endif + /* ** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. ** ** The file used to implement shared-memory is in the same directory @@ -4539,14 +5266,29 @@ /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it ** with the same permissions. */ if( osFstat(pDbFd->h, &sStat) ){ + storeLastErrno(pDbFd, errno); rc = SQLITE_IOERR_FSTAT; goto shm_open_err; } +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE \ + && !defined(SQLITE_SHM_DIRECTORY) + /* If pDbFd is configured with proxy locking mode, use the local + ** lock file path to determine the -shm file path + */ + if( isProxyLockingMode(pDbFd) ){ + zBasePath = proxySharedMemoryBasePath(pDbFd); + if( !zBasePath ){ + rc = SQLITE_CANTOPEN_BKPT; + goto shm_open_err; + } + } +#endif + #ifdef SQLITE_SHM_DIRECTORY nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; #else nShmFilename = 6 + (int)strlen(zBasePath); #endif @@ -4575,10 +5317,17 @@ goto shm_open_err; } } if( pInode->bProcessLock==0 ){ +#ifdef __APPLE__ + /* On MacOS and iOS, avoid even trying to open a read-only SHM file + ** for writing, because doing so generates scary log messages */ + if( osAccess(zShm, R_OK|W_OK)!=0 && (errno==EPERM || errno==EACCES) ){ + pShmNode->hShm = -1; + }else +#endif if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, (sStat.st_mode&0777)); } if( pShmNode->hShm<0 ){ @@ -4976,12 +5725,16 @@ assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->hShm>=0 ){ - osUnlink(pShmNode->zFilename); + if( deleteFlag && pShmNode->hShm>=0 ) { + if (deleteFlag == 1) { + osUnlink(pShmNode->zFilename); + } else if (deleteFlag == 2) { + /* ftruncate(pShmNode->hShm, 32 * 1024); */ + } } unixShmPurge(pDbFd); } unixLeaveMutex(); @@ -4994,10 +5747,418 @@ # define unixShmLock 0 # define unixShmBarrier 0 # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static const char *unixTempFileDir(void); + +static int unixInvalidateSupportFiles(unixFile *pFile, int skipWAL) { + char jPath[MAXPATHLEN+9]; + int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); + if( zLenpInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode && !pShmNode->isReadonly ){ + struct stat sStat; + sqlite3_mutex_enter(pShmNode->pShmMutex); + + if( pShmNode->hShm>=0 && !osFstat(pShmNode->hShm, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + bzero(pShmNode->apRegion[0], size); + sqlite3_mutex_leave(pShmNode->pShmMutex); + unixLeaveMutex(); + continue; + } + } + sqlite3_mutex_leave(pShmNode->pShmMutex); + } + unixLeaveMutex(); + } + jLen = strlcpy(&jPath[zLen], extensions[j], 9); + if( jLen < 9 ){ + int jflags = (j<2) ? O_TRUNC : O_RDWR; + int jfd = open(jPath, jflags); + if( jfd==(-1) ){ + if( errno!=ENOENT ){ + perror(jPath); + } + } else { + if( j==2 ){ + struct stat sStat; + if( !osFstat(jfd, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + uint32_t zero = 0; + pwrite(jfd, &zero, (size_t)size, 0); + } + } + } + fsync(jfd); + close(jfd); + } + } + } + } + return SQLITE_OK; +} + +static int unixUnsafeTruncateDatabase(unixFile *pFile){ + /* this is nasty & bad. destruction with prejudice. we'll lose all the + ** file locks in this process, however. sqlite3_file_control works properly. + ** But if it fails, this works approximately + */ + char journalPath[MAXPATHLEN]; + char walPath[MAXPATHLEN]; + int rc = SQLITE_OK; + +#ifdef DEBUG + fprintf(stderr, "Force truncating database %s\n", pFile->zPath); +#endif + strlcpy(journalPath, pFile->zPath, MAXPATHLEN); + strlcat(journalPath, "-journal", MAXPATHLEN); + strlcpy(walPath, pFile->zPath, MAXPATHLEN); + strlcat(walPath, "-wal", MAXPATHLEN); + int fd1 = pFile->h; + int result = 0; + result = ftruncate(fd1, 0ll); + if (result) { + result = errno; + } + if (result) { + rc = SQLITE_IOERR; + storeLastErrno(pFile, result); + } + + int fd2 = open(journalPath, O_RDWR); + int result2 = 0; + if (fd2 < 0) { + if (errno != ENOENT) { + result2 = errno; + } else { + result2 = 0; + } + } else { + result2 = ftruncate(fd2, 0ll); + if (result2) { + result2 = errno; + } + } + if (result2 && !result) { + rc = SQLITE_IOERR; + storeLastErrno(pFile, result2); + } + + int fd3 = open(walPath, O_RDWR); + int result3 = 0; + if (fd3 < 0) { + if (errno != ENOENT) { + result3 = errno; + } else { + result3 = 0; + } + } else { + result3 = ftruncate(fd3, 0ll); + if (result3) { + result3 = errno; + } + } + if (result3 && !(result || result2)) { + rc = SQLITE_IOERR; + storeLastErrno(pFile, result2); + } + + if (fd3 >= 0) { + fsync(fd3); + close(fd3); + } + if (fd2 >= 0) { + fsync(fd2); + close(fd2); + } + fsync(fd1); + + return rc; +} + +static int unixTruncateDatabase(unixFile *pFile, int bFlags) { + sqlite3_file *id = (sqlite3_file *)pFile; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + int corruptFileLock = 0; + int isCorrupt = 0; + int force = (bFlags & SQLITE_TRUNCATE_FORCE); + int safeFailed = 0; + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, + &corruptFileLock); + } + if( rc && !force ){ + return rc; + } + rc = SQLITE_OK; /* Ignore the locking failure if force is true */ + } + if( (bFlags&SQLITE_TRUNCATE_INITIALIZE_HEADER_MASK)!=0 ){ + /* initialize a new database in TMPDIR and copy the contents over */ + const char *tDir = unixTempFileDir(); + int tDirLen = strlen(tDir); + int tLen = sizeof(char) * (tDirLen + 12); + char *tDbPath = (char *)malloc(tLen); + int tFd = -1; + + strlcpy(tDbPath, tDir, tLen); + if( tDbPath[(tDirLen-1)] != '/' ){ + strlcat(tDbPath, "/tmpdbXXXXX", tLen); + } else { + strlcat(tDbPath, "tmpdbXXXXX", tLen); + } + tFd = mkstemp(tDbPath); + if( tFd==-1 ){ + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR; + safeFailed = 1; + }else{ + sqlite3 *tDb = NULL; + copyfile_state_t s; + int trc = sqlite3_open_v2(tDbPath, &tDb, + (SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_AUTOPROXY), NULL); + char *errmsg = NULL; + const char *sql = ""; + if( !trc && (bFlags&SQLITE_TRUNCATE_PAGESIZE_MASK) ){ + const char pagesize_sql[4][22] = { + "pragma page_size=1024", + "pragma page_size=2048", + "pragma page_size=4096", + "pragma page_size=8192" + }; + int iPagesize = (((bFlags&SQLITE_TRUNCATE_PAGESIZE_MASK) >> 4) - 1); + assert( iPagesize>=0 && iPagesize<=4 ); + sql = pagesize_sql[iPagesize]; + trc = sqlite3_exec(tDb, sql, 0, 0, &errmsg); + } + if( !trc ){ + const char autovacuum_sql[3][21] = { + "pragma auto_vacuum=0", + "pragma auto_vacuum=1", + "pragma auto_vacuum=2" + }; + int iAutovacuum = 2; /* default to incremental */ + if( (bFlags&SQLITE_TRUNCATE_AUTOVACUUM_MASK) ){ + iAutovacuum = (((bFlags&SQLITE_TRUNCATE_AUTOVACUUM_MASK) >> 2) - 1); + } + assert( iAutovacuum>=0 && iAutovacuum<=2 ); + sql = autovacuum_sql[iAutovacuum]; + trc = sqlite3_exec(tDb, sql, 0, 0, &errmsg); + } + if( !trc && (bFlags&SQLITE_TRUNCATE_JOURNALMODE_WAL) ){ + sql = "pragma journal_mode=wal"; + trc = sqlite3_exec(tDb, sql, 0, 0, &errmsg); + } + if( trc ){ + if( !tDb ){ + fprintf(stderr, "failed to open temp database '%s' to reset " + "truncated database %s with flags %x: %d\n", + tDbPath, pFile->zPath, bFlags, trc); + }else{ + fprintf(stderr, "failed to set '%s' on truncated database %s, %d: " + "%s\n", sql, pFile->zPath, trc, errmsg); + } + } + if( tDb ){ + int off = 0; + /* merge the wal into the db */ + sqlite3_file_control(tDb, NULL, SQLITE_FCNTL_PERSIST_WAL, &off); + sqlite3_close(tDb); + } + s = copyfile_state_alloc(); + lseek(tFd, 0, SEEK_SET); + lseek(pFile->h, 0, SEEK_SET); + if( fcopyfile(tFd, pFile->h, s, COPYFILE_DATA) ){ + int err=errno; + switch(err) { + case ENOMEM: + trc = SQLITE_NOMEM; + break; + default: + storeLastErrno(pFile, err); + trc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + fsync(pFile->h); + close(tFd); + unlink(tDbPath); + if( trc!=SQLITE_OK ){ + safeFailed = 1; + rc = trc; + } + } + free(tDbPath); + } else { + rc = pFile->pMethod->xTruncate(id, + ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); + if( rc ){ + safeFailed = 1; + } + } + if( rc==SQLITE_OK || force ){ + rc = unixInvalidateSupportFiles(pFile, 0); + if( rc ){ + safeFailed = 1; + } + } + pFile->pMethod->xSync(id, SQLITE_SYNC_FULL); + + + if( isCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptFileLock); + }else if( pLock ){ + sqlite3demo_superunlock(pLock); + }else{ + assert(force); + } + + if( force && safeFailed){ + rc = unixUnsafeTruncateDatabase(pFile); + } + + return rc; +} + +/* + ** Lock locations for shared-memory locks used by WAL mode. + */ +#ifndef SHM_BASE +# define SHM_BASE 120 +# define SHM_WRITE SHM_BASE +# define SHM_CHECKPOINT (SHM_BASE+1) +# define SHM_RECOVER (SHM_BASE+2) +# define SHM_READ_FIRST (SHM_BASE+3) +# define SHM_READ_SIZE 5 +#endif /* SHM_BASE */ + +/* +** This test only works for lock testing on unix/posix VFS. +** Adapted from tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +*/ +static int unixLockstatePid(unixFile *pFile, pid_t pid, int *pLockstate){ + int hDb; /* File descriptor for the open database file */ + int hShm = -1; /* File descriptor for WAL shared-memory file */ + ssize_t got; /* Bytes read from header */ + int isWal = 0; /* True if in WAL mode */ + int nLock = 0; /* Number of locks held */ + int noHdr = 0; /* Zero byte DB has no header */ + unsigned char aHdr[100]; /* Database header */ + + assert(pLockstate); + + /* make sure we are dealing with a database file */ + hDb = pFile->h; + if( hDb<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); + got = pread(hDb, aHdr, 100, 0); + if( got<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + if( got==0 ){ + noHdr = 1; + }else if( got!=100 + || memcmp(aHdr, SQLITE_FILE_HEADER, SQLITE_FILE_HEADER_LEN)!=0 + ){ + *pLockstate = SQLITE_LOCKSTATE_NOTADB; + return SQLITE_NOTADB; + } + + /* First check for an exclusive lock */ + nLock += unixIsLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, + "EXCLUSIVE"); + if (!noHdr) { + isWal = aHdr[18]==2; + } + if( nLock==0 && isWal==0 ){ + /* Rollback mode */ + nLock += unixIsLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, + "PENDING|RESERVED|SHARED"); + } + if( nLock==0 && isWal!=0 ){ + /* lookup the file descriptor for the shared memory file if we have it open + ** in this process */ + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode ){ + sqlite3_mutex_enter(pShmNode->pShmMutex); + + hShm = pShmNode->hShm; + if( hShm >= 0){ + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + } + + sqlite3_mutex_leave(pShmNode->pShmMutex); + } + + if( hShm<0 ){ + /* the shared memory file isn't open in this process space, open our + ** own FD */ + char zShm[MAXPATHLEN]; + + /* WAL mode */ + strlcpy(zShm, pFile->zPath, MAXPATHLEN); + strlcat(zShm, "-shm", MAXPATHLEN); + hShm = open(zShm, O_RDONLY, 0); + if( hShm<0 ){ + *pLockstate = SQLITE_LOCKSTATE_OFF; + unixLeaveMutex(); + return SQLITE_OK; + } + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + close(hShm); + } + unixLeaveMutex(); + } + if( nLock>0 ){ + *pLockstate = SQLITE_LOCKSTATE_ON; + } else { + *pLockstate = SQLITE_LOCKSTATE_OFF; + } + return SQLITE_OK; +} + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + #if SQLITE_MAX_MMAP_SIZE>0 /* ** If it is currently memory mapped, unmap file pFd. */ static void unixUnmapfile(unixFile *pFd){ @@ -5372,11 +6533,11 @@ static int proxyUnlock(sqlite3_file*, int); static int proxyCheckReservedLock(sqlite3_file*, int*); IOMETHODS( proxyIoFinder, /* Finder function name */ proxyIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ + 2, /* shared memory is enabled */ proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ proxyCheckReservedLock, /* xCheckReservedLock method */ 0 /* xShmMap method */ @@ -5569,10 +6730,12 @@ if( pLockingStyle == &posixIoMethods #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE || pLockingStyle == &nfsIoMethods #endif + /* support WAL mode on read only mounted filesystem */ + || (pLockingStyle == &nolockIoMethods && zFilename!=0) ){ unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ /* If an error occurred in findInodeInfo(), close the file descriptor @@ -5949,11 +7112,15 @@ int *pOutFlags /* Output flags returned to SQLite core */ ){ unixFile *p = (unixFile *)pFile; int fd = -1; /* File descriptor returned by open() */ int openFlags = 0; /* Flags to pass to open() */ - int eType = flags&0x0FFF00; /* Type of file to open */ +#if SQLITE_ENABLE_DATA_PROTECTION + int eType = flags&0xFE0FFF00; /* Type of file to open */ +#else + int eType = flags&0xFEFFFF00; /* Type of file to open */ +#endif int noLock; /* True to omit locking primitives */ int rc = SQLITE_OK; /* Function Return Code */ int ctrlFlags = 0; /* UNIXFILE_* flags */ int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); @@ -6061,10 +7228,14 @@ if( isReadWrite ) openFlags |= O_RDWR; if( isCreate ) openFlags |= O_CREAT; if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); +#if SQLITE_ENABLE_DATA_PROTECTION + p->protFlags = (flags & SQLITE_OPEN_FILEPROTECTION_MASK); +#endif + if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ uid_t uid; /* Userid for the file */ gid_t gid; /* Groupid for the file */ rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); @@ -6135,11 +7306,11 @@ } #else osUnlink(zName); #endif } -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) else{ p->openFlags = openFlags; } #endif @@ -6162,10 +7333,15 @@ if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; if( isNewJrnl ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; +#if defined(SQLITE_ENABLE_PERSIST_WAL) + if( eType==SQLITE_OPEN_MAIN_DB ) { + ctrlFlags |= UNIXFILE_PERSIST_WAL; + } +#endif #if SQLITE_ENABLE_LOCKING_STYLE #if SQLITE_PREFER_PROXY_LOCKING isAutoProxy = 1; #endif @@ -6181,17 +7357,23 @@ useProxy = !(fsInfo.f_flags&MNT_LOCAL); } if( useProxy ){ rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); if( rc==SQLITE_OK ){ + /* cache the pMethod in case the transform fails */ + const struct sqlite3_io_methods *pMethod = pFile->pMethods; rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); if( rc!=SQLITE_OK ){ /* Use unixClose to clean up the resources added in fillInUnixFile ** and clear all the structure's references. Specifically, ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op */ - unixClose(pFile); + if( pMethod!=NULL ){ + pMethod->xClose(pFile); + }else{ + unixClose(pFile); + } return rc; } } goto open_finished; } @@ -6241,11 +7423,17 @@ rc = osOpenDirectory(zPath, &fd); if( rc==SQLITE_OK ){ if( full_fsync(fd,0,0) ){ rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } +#if OSCLOSE_CHECK_CLOSE_IOERR + if( close(fd)&&!rc ){ + rc = SQLITE_IOERR_DIR_CLOSE; + } +#else robust_close(0, fd, __LINE__); +#endif }else{ assert( rc==SQLITE_CANTOPEN ); rc = SQLITE_OK; } } @@ -6872,10 +8060,36 @@ } OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); return 0; } +#if SQLITE_ENABLE_LOCKING_STYLE +static int isProxyLockingMode(unixFile *pFile) { + return (pFile->pMethod == &proxyIoMethods) ? 1 : 0; +} +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** Return the shared memory base path based on the lock proxy file if the +** lock proxy file is hosted on a shared memory compatible FS +*/ +static const char *proxySharedMemoryBasePath(unixFile *pFile) { + proxyLockingContext *pCtx; + unixFile *pLockFile; + + assert(pFile!=NULL && pFile->lockingContext!=NULL); + assert(pFile->pMethod == &proxyIoMethods); + pCtx = ((proxyLockingContext *)(pFile->lockingContext)); + pLockFile = pCtx->lockProxy; + if( pLockFile->pMethod->iVersion>=2 && pLockFile->pMethod->xShmMap!=0 ){ + return pCtx->lockProxyPath; + } + return NULL; +} +#endif + /* ** Create a new VFS file descriptor (stored in memory obtained from ** sqlite3_malloc) and open the file named "path" in the file descriptor. ** ** The caller is responsible not only for closing the file descriptor @@ -6922,10 +8136,11 @@ openFlags = O_RDONLY | O_NOFOLLOW; fd = robust_open(path, openFlags, 0); terrno = errno; } if( fd<0 ){ + sqlite3_free(pUnused); if( islockfile ){ return SQLITE_BUSY; } switch (terrno) { case EACCES: @@ -7080,18 +8295,17 @@ */ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *conchFile = pCtx->conchFile; int rc = SQLITE_OK; - int nTries = 0; struct timespec conchModTime; memset(&conchModTime, 0, sizeof(conchModTime)); do { rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); - nTries ++; if( rc==SQLITE_BUSY ){ + pCtx->nFails ++; /* If the lock failed (busy): * 1st try: get the mod time of the conch, wait 0.5s and try again. * 2nd try: fail if the mod time changed or host id is different, wait * 10 sec and try again * 3rd try: break the lock unless the mod time has changed. @@ -7100,32 +8314,48 @@ if( osFstat(conchFile->h, &buf) ){ storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } - if( nTries==1 ){ + if( pCtx->nFails==1 ){ conchModTime = buf.st_mtimespec; usleep(500000); /* wait 0.5 sec and try the lock again*/ continue; } - assert( nTries>1 ); + assert( pCtx->nFails>1 ); if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ return SQLITE_BUSY; } - if( nTries==2 ){ + if( pCtx->nFails==2 ){ char tBuf[PROXY_MAXCONCHLEN]; int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ - /* don't break the lock if the host id doesn't match */ + /* don't break the lock if the host id doesn't match, but do log + * an error to console so users can diagnose stale NFS locks more + * easily + */ if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + uuid_t conchUUID; + uuid_string_t conchUUIDString; + uuid_string_t myUUIDString; + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memcpy(conchUUID, &tBuf[PROXY_HEADERLEN], PROXY_HOSTIDLEN); + uuid_unparse(conchUUID, conchUUIDString); + uuid_unparse(myHostID, myUUIDString); + fprintf(stderr, + "ERROR: sqlite database is locked because it is in use " + "by another host that holds a host-exclusive lock on %s; " + "this host (UUID %s) cannot override the host-exclusive lock " + "until the other host (UUID %s) releases its locks on %s\n", + pFile->zPath, myUUIDString, conchUUIDString, conchFile->zPath); return SQLITE_BUSY; } }else{ /* don't break the lock on short read or a version mismatch */ return SQLITE_BUSY; @@ -7132,22 +8362,22 @@ } usleep(10000000); /* wait 10 sec and try the lock again */ continue; } - assert( nTries==3 ); - if( 0==proxyBreakConchLock(pFile, myHostID) ){ + assert( pCtx->nFails>=3 ); + if( (pCtx->nFails==3)&&(0==proxyBreakConchLock(pFile, myHostID)) ){ rc = SQLITE_OK; if( lockType==EXCLUSIVE_LOCK ){ rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); } if( !rc ){ rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); } } } - } while( rc==SQLITE_BUSY && nTries<3 ); + } while( rc==SQLITE_BUSY && pCtx->nFails<3 ); return rc; } /* Takes the conch by taking a shared lock and read the contents conch, if @@ -7319,11 +8549,18 @@ end_takeconch: OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ int fd; if( pFile->h>=0 ){ +#if defined(STRICT_CLOSE_ERROR) && OSCLOSE_CHECK_CLOSE_IOERR + if( close(pFile->h) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_CLOSE; + } +#else robust_close(pFile, pFile->h, __LINE__); +#endif } pFile->h = -1; fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ @@ -7562,10 +8799,13 @@ } } } if( rc==SQLITE_OK && lockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); + if( pCtx->lockProxyPath==NULL ){ + rc = SQLITE_NOMEM; + } } if( rc==SQLITE_OK ){ pCtx->dbPath = sqlite3DbStrDup(0, dbPath); if( pCtx->dbPath==NULL ){ @@ -7601,11 +8841,11 @@ */ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ switch( op ){ case SQLITE_FCNTL_GET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; - if( pFile->pMethod == &proxyIoMethods ){ + if( isProxyLockingMode(pFile) ){ proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; proxyTakeConch(pFile); if( pCtx->lockProxyPath ){ *(const char **)pArg = pCtx->lockProxyPath; }else{ @@ -7617,11 +8857,11 @@ return SQLITE_OK; } case SQLITE_FCNTL_SET_LOCKPROXYFILE: { unixFile *pFile = (unixFile*)id; int rc = SQLITE_OK; - int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + int isProxyStyle = isProxyLockingMode(pFile); if( pArg==NULL || (const char *)pArg==0 ){ if( isProxyStyle ){ /* turn off proxy locking - not supported. If support is added for ** switching proxy locking mode off then it will need to fail if ** the journal mode is WAL mode. Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3541,10 +3541,19 @@ OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; + } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + *(int*)pArg = pFile->bPersistWal; + }else{ + pFile->bPersistWal = bPersist!=0; + } + return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -2499,11 +2499,15 @@ pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); if( !pMaster ){ rc = SQLITE_NOMEM_BKPT; }else{ - const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); + const int flags = +#if SQLITE_ENABLE_DATA_PROTECTION + (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK)| +#endif + (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); } if( rc!=SQLITE_OK ) goto delmaster_out; /* Load the entire master journal file into space obtained from @@ -2536,11 +2540,15 @@ /* One of the journals pointed to by the master journal exists. ** Open it and check if it points at the master journal. If ** so, return without deleting the master journal file. */ int c; - int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL); + int flags = +#if SQLITE_ENABLE_DATA_PROTECTION + (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK)| +#endif + (SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL); rc = sqlite3OsOpen(pVfs, zJournal, pJournal, flags, 0); if( rc!=SQLITE_OK ){ goto delmaster_out; } @@ -3607,11 +3615,15 @@ #ifdef SQLITE_TEST sqlite3_opentemp_count++; /* Used for testing and analysis only */ #endif - vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + vfsFlags |= +#if SQLITE_ENABLE_DATA_PROTECTION + (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK)| +#endif + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); assert( rc!=SQLITE_OK || isOpen(pFile) ); return rc; } @@ -5091,11 +5103,15 @@ ** at least one non-zero bytes at the start of the journal file. ** If there is, then we consider this journal to be hot. If not, ** it can be ignored. */ if( !jrnlOpen ){ - int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; + int f = +#if SQLITE_ENABLE_DATA_PROTECTION + (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK)| +#endif + SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); } if( rc==SQLITE_OK ){ u8 first = 0; rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); @@ -5231,11 +5247,15 @@ int bExists; /* True if journal file exists */ rc = sqlite3OsAccess( pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); if( rc==SQLITE_OK && bExists ){ int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + int f = +#if SQLITE_ENABLE_DATA_PROTECTION + (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK)| +#endif + SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; assert( !pPager->tempFile ); rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ rc = SQLITE_CANTOPEN_BKPT; @@ -5738,10 +5758,13 @@ sqlite3MemJournalOpen(pPager->jfd); }else{ int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; int nSpill; +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= (pPager->vfsFlags&SQLITE_OPEN_FILEPROTECTION_MASK); +#endif if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; @@ -6976,10 +6999,22 @@ ** not yet been opened. */ sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } + +#if !defined(SQLITE_OMIT_WAL) +/* + ** Return the file handle for the WAL journal file associated + ** with the pager. This might return NULL if the file has + ** not yet been opened. + */ +sqlite3_file *sqlite3PagerWalFile(Pager *pPager){ + return ((pPager->pWal) ? sqlite3WalFile(pPager->pWal) : (NULL)); +} +#endif + #ifdef SQLITE_ENABLE_SETLK_TIMEOUT /* ** Reset the lock timeout for pager. */ @@ -7462,14 +7497,18 @@ /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, &pPager->pWal - ); +#if SQLITE_ENABLE_DATA_PROTECTION + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, (pPager->vfsFlags & (SQLITE_OPEN_FILEPROTECTION_MASK | SQLITE_OPEN_READONLY)), + &pPager->pWal); +#else + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, (pPager->vfsFlags & SQLITE_OPEN_READONLY), &pPager->pWal); +#endif } pagerFixMaplimit(pPager); return rc; } @@ -7559,10 +7598,17 @@ if( rc==SQLITE_OK ){ rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); + + /* Ensure that the WAL file is deleted even if the PERSIST_WAL + ** hint is enabled. */ + if( rc==SQLITE_OK ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; + } if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } } return rc; } Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -201,10 +201,11 @@ #endif int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(const Pager*, int); sqlite3_vfs *sqlite3PagerVfs(Pager*); sqlite3_file *sqlite3PagerFile(Pager*); +sqlite3_file *sqlite3PagerWalFile(Pager *pPager); sqlite3_file *sqlite3PagerJrnlFile(Pager*); const char *sqlite3PagerJournalname(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); void sqlite3PagerCacheStat(Pager *, int, int, int *); Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -384,12 +384,11 @@ }else{ assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS { - int nFreed = 0; - nFreed = sqlite3MallocSize(p); + int nFreed = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_OVERFLOW, nFreed); sqlite3_mutex_leave(pcache1.mutex); } #endif Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -215,15 +215,15 @@ assert( SQLITE_FullFSync==PAGER_FULLFSYNC ); assert( SQLITE_CkptFullFSync==PAGER_CKPT_FULLFSYNC ); assert( SQLITE_CacheSpill==PAGER_CACHESPILL ); assert( (PAGER_FULLFSYNC | PAGER_CKPT_FULLFSYNC | PAGER_CACHESPILL) == PAGER_FLAGS_MASK ); - assert( (pDb->safety_level & PAGER_SYNCHRONOUS_MASK)==pDb->safety_level ); while( (n--) > 0 ){ if( pDb->pBt ){ sqlite3BtreeSetPagerFlags(pDb->pBt, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK) ); + (pDb->safety_level & PAGER_SYNCHRONOUS_MASK) + | (db->flags & PAGER_FLAGS_MASK) ); } pDb++; } } } Index: src/prepare.c ================================================================== --- src/prepare.c +++ src/prepare.c @@ -12,10 +12,13 @@ ** This file contains the implementation of the sqlite3_prepare() ** interface, and routines that contribute to loading the database schema ** from disk. */ #include "sqliteInt.h" +#ifdef SQLITE_ENABLE_SQLRR +# include "sqlrr.h" +#endif /* ** Fill the InitData structure with an error message that indicates ** that the database is corrupt. */ @@ -776,10 +779,13 @@ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ int rc; rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); +#ifdef SQLITE_ENABLE_SQLRR + SRRecPrepare(db, zSql, nBytes, 0, *ppStmt); +#endif assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } int sqlite3_prepare_v2( sqlite3 *db, /* Database handle. */ @@ -816,10 +822,13 @@ ** Proof by comparison to the implementation of sqlite3_prepare_v2() ** directly above. */ rc = sqlite3LockAndPrepare(db,zSql,nBytes, SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), 0,ppStmt,pzTail); +#ifdef SQLITE_ENABLE_SQLRR + SRRecPrepare(db, zSql, nBytes, 1, *ppStmt); +#endif assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); return rc; } @@ -859,11 +868,13 @@ sqlite3_mutex_enter(db->mutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8); } - +#ifdef SQLITE_ENABLE_SQLRR + SRRecPrepare(db, zSql8, -1, 1, *ppStmt); +#endif if( zTail8 && pzTail ){ /* If sqlite3_prepare returns a tail pointer, we calculate the ** equivalent pointer into the UTF-16 string by counting the unicode ** characters between zSql8 and zTail8, and then returning a pointer ** the same number of characters into the UTF-16 string. Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -569,12 +569,13 @@ #define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_FILEPROTECTION_MASK \ + 0x00700000 #define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ - /* Reserved: 0x00F00000 */ /* ** CAPI3REF: Device Characteristics ** ADDED src/sqlite3_private.h Index: src/sqlite3_private.h ================================================================== --- /dev/null +++ src/sqlite3_private.h @@ -0,0 +1,80 @@ +/* + * sqlite3_private.h + */ + +#ifndef _SQLITE3_PRIVATE_H +#define _SQLITE3_PRIVATE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define SQLITE_LOCKSTATE_OFF 0 +#define SQLITE_LOCKSTATE_ON 1 +#define SQLITE_LOCKSTATE_NOTADB 2 +#define SQLITE_LOCKSTATE_ERROR -1 + +#define SQLITE_LOCKSTATE_ANYPID -1 + +/* +** Test a file path for sqlite locks held by a process ID (-1 = any PID). +** Returns one of the following integer codes: +** +** SQLITE_LOCKSTATE_OFF no active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_ON active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_NOTADB path points to a file that is not an sqlite db file +** SQLITE_LOCKSTATE_ERROR path was not vaild or was unreadable +** +** There is no support for identifying db files encrypted via SEE encryption +** currently. Zero byte files are tested for sqlite locks, but if no sqlite +** locks are present then SQLITE_LOCKSTATE_NOTADB is returned. +*/ +extern int _sqlite3_lockstate(const char *path, pid_t pid); + +/* +** Test an open database connection for sqlite locks held by a process ID, +** if a process has an open database connection this will avoid trashing file +** locks by re-using open file descriptors for the database file and support +** files (-shm) +*/ +#define SQLITE_FCNTL_LOCKSTATE_PID 103 + +/* +** Pass the SQLITE_TRUNCATE_DATABASE operation code to sqlite3_file_control() +** to truncate a database and its associated journal file to zero length. The +** SQLITE_TRUNCATE_* flags represent optional flags to safely initialize an +** empty database in the place of the truncated database, the flags are passed +** into sqlite3_file_control via the fourth argument using a pointer to an integer +** configured with the ORed flags. If the fourth argument is NULL, the default +** behavior is applied and the database file is truncated to zero bytes, a rollback +** journal (if present) is unlinked, a WAL journal (if present) is truncated to zero +** bytes and the first few bytes of the -shm file is scrambled to trigger existing +** connections to rebuild the index from the database file contents. +*/ +#define SQLITE_FCNTL_TRUNCATE_DATABASE 101 +#define SQLITE_TRUNCATE_DATABASE SQLITE_FCNTL_TRUNCATE_DATABASE +#define SQLITE_TRUNCATE_INITIALIZE_HEADER_MASK (0x7F<<0) +#define SQLITE_TRUNCATE_JOURNALMODE_WAL (0x1<<0) +#define SQLITE_TRUNCATE_AUTOVACUUM_MASK (0x3<<2) +#define SQLITE_TRUNCATE_AUTOVACUUM_OFF (0x1<<2) +#define SQLITE_TRUNCATE_AUTOVACUUM_FULL (0x2<<2) +#define SQLITE_TRUNCATE_AUTOVACUUM_INCREMENTAL (0x3<<2) +#define SQLITE_TRUNCATE_PAGESIZE_MASK (0x7<<4) +#define SQLITE_TRUNCATE_PAGESIZE_1024 (0x1<<4) +#define SQLITE_TRUNCATE_PAGESIZE_2048 (0x2<<4) +#define SQLITE_TRUNCATE_PAGESIZE_4096 (0x3<<4) +#define SQLITE_TRUNCATE_PAGESIZE_8192 (0x4<<4) +#define SQLITE_TRUNCATE_FORCE (0x1<<7) + +/* +** Pass the SQLITE_REPLACE_DATABASE operation code to sqlite3_file_control() +** and a sqlite3 pointer to another open database file to safely copy the +** contents of that database file into the receiving database. +*/ +#define SQLITE_FCNTL_REPLACE_DATABASE 102 +#define SQLITE_REPLACE_DATABASE SQLITE_FCNTL_REPLACE_DATABASE + +#ifdef __cplusplus +} /* End of the 'extern "C"' block */ +#endif +#endif Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -344,10 +344,26 @@ + defined(SQLITE_ZERO_MALLOC) \ + defined(SQLITE_MEMDEBUG)==0 # define SQLITE_SYSTEM_MALLOC 1 #endif +/* +** This is a magic string that appears at the beginning of every +** SQLite database in order to identify the file as a real database. +** +** You can change this value at compile-time by specifying a +** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The +** header must be exactly 16 bytes including the zero-terminator so +** the string itself should be 15 characters long. If you change +** the header, then your custom library will not be able to read +** databases generated by the standard tools and the standard tools +** will not be able to read databases created by your custom library. +*/ +#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ +# define SQLITE_FILE_HEADER "SQLite format 3" +#endif + /* ** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the ** sizes of memory allocations below this value where possible. */ #if !defined(SQLITE_MALLOC_SOFT_LIMIT) @@ -4920,10 +4936,34 @@ #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ #define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ #define MEMTYPE_PCACHE 0x04 /* Page cache allocations */ +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + +/* +** An instance of the following structure is used to hold the process ID +** and return-by-reference lockstate value. The SQLITE_FCNTL_LOCKSTATE_PID +** requires the 4th argument to sqlite3_file_control to be a pointer to an +** instance of LockstatePID initialized with a LockstatePID.pid value equal +** to a process ID to be tested, or the special value SQLITE_LOCKSTATE_ANYPID +** The Lockstate.state value is always set to one of the following values +** when sqlite3_file_control returns: +** +** SQLITE_LOCKSTATE_OFF no active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_ON active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_NOTADB path points to a file that is not an sqlite db file +** SQLITE_LOCKSTATE_ERROR path was not vaild or was unreadable +*/ +typedef struct LockstatePID LockstatePID; +struct LockstatePID { + pid_t pid; /* Process ID to test */ + int state; /* The state of the lock (return value) */ +}; + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + /* ** Threading interface */ #if SQLITE_MAX_WORKER_THREADS>0 int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -4048,10 +4048,28 @@ } if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3_clear_bindings(pStmt))); return TCL_OK; } + +/* + ** Usage: sqlite3_clear_bindings STMT + ** + */ +static int test_clear_bindings_null( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( objc!=1 ){ + return TCL_ERROR; + } + /* test for handling NULL */ + Tcl_SetObjResult(interp, Tcl_NewIntObj(sqlite3_clear_bindings(0))); + return TCL_OK; +} /* ** Usage: sqlite3_sleep MILLISECONDS */ static int SQLITE_TCLAPI test_sleep( @@ -5893,10 +5911,100 @@ Tcl_SetResult(interp, (char *)zBuf, TCL_VOLATILE); return TCL_OK; } } +#ifdef __APPLE__ +/* From sqlite3_private.h */ +# ifndef SQLITE_TRUNCATE_DATABASE +# define SQLITE_TRUNCATE_DATABASE 101 +# define SQLITE_TRUNCATE_JOURNALMODE_WAL (0x1<<0) +# define SQLITE_TRUNCATE_AUTOVACUUM_MASK (0x3<<2) +# define SQLITE_TRUNCATE_AUTOVACUUM_OFF (0x1<<2) +# define SQLITE_TRUNCATE_AUTOVACUUM_FULL (0x2<<2) +# define SQLITE_TRUNCATE_AUTOVACUUM_INCREMENTAL (0x3<<2) +# define SQLITE_TRUNCATE_PAGESIZE_MASK (0x7<<4) +# define SQLITE_TRUNCATE_PAGESIZE_1024 (0x1<<4) +# define SQLITE_TRUNCATE_PAGESIZE_2048 (0x2<<4) +# define SQLITE_TRUNCATE_PAGESIZE_4096 (0x3<<4) +# define SQLITE_TRUNCATE_PAGESIZE_8192 (0x4<<4) +# endif +# ifndef SQLITE_REPLACE_DATABASE +# define SQLITE_REPLACE_DATABASE 102 +# endif + +/* +** tclcmd: file_control_truncate_test DB +** +** This TCL command runs the sqlite3_file_control interface and +** verifies correct operation of the SQLITE_TRUNCATE_DATABASE verb. +*/ +static int file_control_truncate_test( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int flags; + int rc; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB FLAGS", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &flags) ){ + return TCL_ERROR; + } + rc = sqlite3_file_control(db, NULL, SQLITE_TRUNCATE_DATABASE, &flags); + if( rc ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** tclcmd: file_control_replace_test DB +** +** This TCL command runs the sqlite3_file_control interface and +** verifies correct operation of the SQLITE_REPLACE_DATABASE verb. +*/ +static int file_control_replace_test( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *src_db; + sqlite3 *dst_db; + int rc; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DST_DB SRC_DB", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &dst_db) ){ + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[2]), &src_db) ){ + return TCL_ERROR; + } + rc = sqlite3_file_control(dst_db, NULL, SQLITE_REPLACE_DATABASE, src_db); + if( rc ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); + return TCL_ERROR; + } + return TCL_OK; +} +#endif /* __APPLE__ */ + /* ** tclcmd: file_control_chunksize_test DB DBNAME SIZE ** ** This TCL command runs the sqlite3_file_control interface and ** verifies correct operation of the SQLITE_GET_LOCKPROXYFILE and @@ -6037,10 +6145,94 @@ } } #endif return TCL_OK; } + +#ifdef __APPLE__ +#include +#include +#include +#endif + +/* + ** tclcmd: path_is_local PWD + */ +static int path_is_local( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ +#ifdef __APPLE__ + const char *zPath; + int nPath; + struct statfs fsInfo; + + if( objc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " PATH", 0); + return TCL_ERROR; + } + zPath = Tcl_GetStringFromObj(objv[1], &nPath); + if( statfs(zPath, &fsInfo) == -1 ){ + int err = errno; + Tcl_AppendResult(interp, "Error calling statfs on path", + Tcl_NewIntObj(err), 0); + return TCL_ERROR; + } + if( fsInfo.f_flags&MNT_LOCAL ){ + Tcl_SetObjResult(interp, Tcl_NewIntObj(1)); + } else { + Tcl_SetObjResult(interp, Tcl_NewIntObj(0)); + } +#else + Tcl_SetObjResult(interp, Tcl_NewIntObj(1)); +#endif + + return TCL_OK; +} + +/* + ** tclcmd: path_is_dos PWD + */ +static int path_is_dos( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ +#ifdef __APPLE__ + const char *zPath; + int nPath; + struct statfs fsInfo; + + if( objc!=2 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " PATH", 0); + return TCL_ERROR; + } + zPath = Tcl_GetStringFromObj(objv[1], &nPath); + if( statfs(zPath, &fsInfo) == -1 ){ + int err = errno; + Tcl_AppendResult(interp, "Error calling statfs on path", + Tcl_NewIntObj(err), 0); + return TCL_ERROR; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + Tcl_SetObjResult(interp, Tcl_NewIntObj(1)); + } else if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { + Tcl_SetObjResult(interp, Tcl_NewIntObj(1)); + } else { + Tcl_SetObjResult(interp, Tcl_NewIntObj(0)); + } +#else + Tcl_SetObjResult(interp, Tcl_NewIntObj(0)); +#endif + + return TCL_OK; +} #if SQLITE_OS_WIN /* ** tclcmd: file_control_win32_av_retry DB NRETRY DELAY ** @@ -6171,11 +6363,10 @@ rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, (void*)&bPersist); sqlite3_snprintf(sizeof(z), z, "%d %d", rc, bPersist); Tcl_AppendResult(interp, z, (char*)0); return TCL_OK; } - /* ** tclcmd: file_control_powersafe_overwrite DB PSOW-FLAG ** ** This TCL command runs the sqlite3_file_control interface with ** the SQLITE_FCNTL_POWERSAFE_OVERWRITE opcode. @@ -7942,10 +8133,11 @@ { "sqlite3_bind_blob", test_bind_blob ,0 }, { "sqlite3_bind_parameter_count", test_bind_parameter_count, 0}, { "sqlite3_bind_parameter_name", test_bind_parameter_name, 0}, { "sqlite3_bind_parameter_index", test_bind_parameter_index, 0}, { "sqlite3_clear_bindings", test_clear_bindings, 0}, + { "sqlite3_clear_bindings_null", test_clear_bindings_null, 0}, { "sqlite3_sleep", test_sleep, 0}, { "sqlite3_errcode", test_errcode ,0 }, { "sqlite3_extended_errcode", test_ex_errcode ,0 }, { "sqlite3_errmsg", test_errmsg ,0 }, { "sqlite3_errmsg16", test_errmsg16 ,0 }, @@ -8057,10 +8249,14 @@ { "vfs_unregister_all", vfs_unregister_all, 0 }, { "vfs_reregister_all", vfs_reregister_all, 0 }, { "file_control_test", file_control_test, 0 }, { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, +#ifdef __APPLE__ + { "file_control_truncate_test", file_control_truncate_test, 0 }, + { "file_control_replace_test", file_control_replace_test, 0 }, +#endif { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "file_control_sizehint_test", file_control_sizehint_test, 0 }, { "file_control_data_version", file_control_data_version, 0 }, #if SQLITE_OS_WIN { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, @@ -8071,10 +8267,12 @@ { "file_control_powersafe_overwrite",file_control_powersafe_overwrite,0}, { "file_control_vfsname", file_control_vfsname, 0 }, { "file_control_tempfilename", file_control_tempfilename, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, + { "path_is_local", path_is_local, 0 }, + { "path_is_dos", path_is_dos, 0 }, /* Functions from os.h */ #ifndef SQLITE_OMIT_UTF16 { "add_test_collate", test_collate, 0 }, { "add_test_collate_needed", test_collate_needed, 0 }, Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -608,11 +608,25 @@ #if defined(SQLITE_PREFER_PROXY_LOCKING) && defined(__APPLE__) Tcl_SetVar2(interp,"sqlite_options","prefer_proxy_locking","1",TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp,"sqlite_options","prefer_proxy_locking","0",TCL_GLOBAL_ONLY); #endif - +#if defined(SQLITE_ENABLE_PURGEABLE_PCACHE) && defined(__APPLE__) + Tcl_SetVar2(interp,"sqlite_options","enable_purgeable_pcache","1",TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp,"sqlite_options","enable_purgeable_pcache","0",TCL_GLOBAL_ONLY); +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + Tcl_SetVar2(interp,"sqlite_options","default_ckptfullfsync","1",TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp,"sqlite_options","default_ckptfullfsync","0",TCL_GLOBAL_ONLY); +#endif +#if SQLITE_ENABLE_PERSIST_WAL + Tcl_SetVar2(interp,"sqlite_options","enable_persist_wal","1",TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp,"sqlite_options","enable_persist_wal","0",TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_SHARED_CACHE Tcl_SetVar2(interp, "sqlite_options", "shared_cache", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "shared_cache", "1", TCL_GLOBAL_ONLY); @@ -743,10 +757,26 @@ #ifdef SQLITE_ENABLE_SQLLOG Tcl_SetVar2(interp, "sqlite_options", "sqllog", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "sqllog", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef __APPLE__ +# if defined(__ppc__) + Tcl_SetVar2(interp, "os_options", "arch", "ppc", TCL_GLOBAL_ONLY); +# elif defined(__i386__) + Tcl_SetVar2(interp, "os_options", "arch", "i386", TCL_GLOBAL_ONLY); +# elif defined(__x86_64__) + Tcl_SetVar2(interp, "os_options", "arch", "x86_64", TCL_GLOBAL_ONLY); +# elif defined(__arm__) + Tcl_SetVar2(interp, "os_options", "arch", "arm", TCL_GLOBAL_ONLY); +# else +# error Unrecognized architecture for exec_options +# endif +#else + Tcl_SetVar2(interp, "os_options", "arch", "unknown", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_ENABLE_URI_00_ERROR Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "uri_00_error", "0", TCL_GLOBAL_ONLY); Index: src/test_superlock.c ================================================================== --- src/test_superlock.c +++ src/test_superlock.c @@ -146,11 +146,11 @@ /* ** Release a superlock held on a database file. The argument passed to ** this function must have been obtained from a successful call to ** sqlite3demo_superlock(). */ -void sqlite3demo_superunlock(void *pLock){ +static void sqlite3demo_superunlock(void *pLock){ Superlock *p = (Superlock *)pLock; if( p->bWal ){ int rc; /* Return code */ int flags = SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE; sqlite3_file *fd = 0; @@ -177,13 +177,14 @@ ** to this function is not NULL, then xBusy is invoked in the same way ** as a busy-handler registered with SQLite (using sqlite3_busy_handler()) ** until either the lock can be obtained or the busy-handler function returns ** 0 (indicating "give up"). */ -int sqlite3demo_superlock( +static int sqlite3demo_superlock( const char *zPath, /* Path to database file to lock */ const char *zVfs, /* VFS to use to access database file */ + int flags, /* Additional flags to pass to sqlite3_open_v2 */ int (*xBusy)(void*,int), /* Busy handler callback */ void *pBusyArg, /* Context arg for busy handler */ void **ppLock /* OUT: Context to pass to superunlock() */ ){ SuperlockBusy busy = {0, 0, 0}; /* Busy handler wrapper object */ @@ -194,11 +195,11 @@ if( !pLock ) return SQLITE_NOMEM; memset(pLock, 0, sizeof(Superlock)); /* Open a database handle on the file to superlock. */ rc = sqlite3_open_v2( - zPath, &pLock->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, zVfs + zPath, &pLock->db, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|flags, zVfs ); /* Install a busy-handler and execute a BEGIN EXCLUSIVE. If this is not ** a WAL database, this is all we need to do. ** @@ -336,11 +337,11 @@ busy.interp = interp; busy.pScript = objv[4]; xBusy = superlock_busy; } - rc = sqlite3demo_superlock(zPath, zVfs, xBusy, &busy, &pLock); + rc = sqlite3demo_superlock(zPath, zVfs, 0, xBusy, &busy, &pLock); assert( rc==SQLITE_OK || pLock==0 ); assert( rc!=SQLITE_OK || pLock!=0 ); if( rc!=SQLITE_OK ){ extern const char *sqlite3ErrStr(int); Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -699,11 +699,12 @@ if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ goto no_mem; } - assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY + || (p->rc&0xFF)==SQLITE_LOCKED ); assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); p->pResultSet = 0; db->busyHandler.nBusy = 0; Index: src/vdbeapi.c ================================================================== --- src/vdbeapi.c +++ src/vdbeapi.c @@ -13,10 +13,13 @@ ** This file contains code use to implement APIs that are part of the ** VDBE. */ #include "sqliteInt.h" #include "vdbeInt.h" +#ifdef SQLITE_ENABLE_SQLRR +# include "sqlrr.h" +#endif #ifndef SQLITE_OMIT_DEPRECATED /* ** Return TRUE (non-zero) of the statement supplied as an argument needs ** to be recompiled. A statement needs to be recompiled whenever the @@ -102,10 +105,13 @@ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL ** pointer is a harmless no-op. */ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; +#ifdef SQLITE_ENABLE_SQLRR + SRRecFinalize(pStmt); +#endif sqlite3 *db = v->db; if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); rc = sqlite3VdbeFinalize(v); @@ -128,10 +134,13 @@ if( pStmt==0 ){ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; sqlite3 *db = v->db; +#ifdef SQLITE_ENABLE_SQLRR + SRRecReset(pStmt); +#endif sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); rc = sqlite3VdbeReset(v); sqlite3VdbeRewind(v); assert( (rc & (db->errMask))==rc ); @@ -147,11 +156,18 @@ int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int i; int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; + sqlite3_mutex *mutex=NULL; +#endif + if( NULL==pStmt ){ return SQLITE_OK; } /* */ +#ifdef SQLITE_ENABLE_SQLRR + SRRecClearBindings(pStmt); +#endif +#if SQLITE_THREADSAFE + mutex = ((Vdbe*)pStmt)->db->mutex; #endif sqlite3_mutex_enter(mutex); for(i=0; inVar; i++){ sqlite3VdbeMemRelease(&p->aVar[i]); p->aVar[i].flags = MEM_Null; @@ -1412,10 +1428,13 @@ int i, const void *zData, int nData, void (*xDel)(void*) ){ +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindBlob(pStmt, i, zData, nData); +#endif #ifdef SQLITE_ENABLE_API_ARMOR if( nData<0 ) return SQLITE_MISUSE_BKPT; #endif return bindText(pStmt, i, zData, nData, xDel, 0); } @@ -1434,23 +1453,32 @@ } } int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindDouble(pStmt, i, rValue); +#endif rc = vdbeUnbind(p, i); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); } return rc; } int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindInt64(p, i, (i64)iValue); +#endif return sqlite3_bind_int64(p, i, (i64)iValue); } int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindInt64(pStmt, i, iValue); +#endif rc = vdbeUnbind(p, i); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); } @@ -1457,10 +1485,13 @@ return rc; } int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindNull(pStmt, i); +#endif rc = vdbeUnbind(p, i); if( rc==SQLITE_OK ){ sqlite3_mutex_leave(p->db->mutex); } return rc; @@ -1488,10 +1519,13 @@ int i, const char *zData, int nData, void (*xDel)(void*) ){ +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindText(pStmt, i, zData, nData); +#endif return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } int sqlite3_bind_text64( sqlite3_stmt *pStmt, int i, @@ -1514,10 +1548,13 @@ int i, const void *zData, int nData, void (*xDel)(void*) ){ +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindText(pStmt, i, zData, nData); +#endif return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; @@ -1537,10 +1574,13 @@ rc = sqlite3_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE_TRANSIENT); } break; } case SQLITE_TEXT: { +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindText(pStmt, i, pValue->z, pValue->n); +#endif rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, pValue->enc); break; } default: { @@ -1551,10 +1591,13 @@ return rc; } int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_SQLRR + SRRecBindBlob(pStmt, i, NULL, n); +#endif rc = vdbeUnbind(p, i); if( rc==SQLITE_OK ){ sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); sqlite3_mutex_leave(p->db->mutex); } Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -1332,15 +1332,16 @@ sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ i64 mxWalSize, /* Truncate WAL to this size on reset */ + int flags, /* VFS file protection flags */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ Wal *pRet; /* Object to allocate and return */ - int flags; /* Flags passed to OsOpen() */ + int vfsFlags; /* Flags passed to OsOpen() */ assert( zWalName && zWalName[0] ); assert( pDbFd ); /* In the amalgamation, the os_unix.c and os_win.c source files come before @@ -1375,13 +1376,17 @@ pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ - flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); - if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + if( flags&SQLITE_OPEN_READONLY ){ + vfsFlags = flags | SQLITE_OPEN_WAL; + } else { + vfsFlags = flags | (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + } + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, vfsFlags, &vfsFlags); + if( rc==SQLITE_OK && vfsFlags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); @@ -2087,10 +2092,13 @@ ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from ** reordering the reads and writes. */ aHdr = walIndexHdr(pWal); + if( aHdr==NULL ){ + return 1; /* Shouldn't be getting NULL from walIndexHdr, but we are */ + } memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ @@ -3002,11 +3010,10 @@ return pWal->hdr.nPage; } return 0; } - /* ** This function starts a write transaction on the WAL. ** ** A read transaction must have already been started by a prior call ** to sqlite3WalBeginReadTransaction(). @@ -3220,10 +3227,14 @@ Wal *pWal; /* The complete WAL information */ sqlite3_file *pFd; /* The WAL file to which we write */ sqlite3_int64 iSyncPoint; /* Fsync at this offset */ int syncFlags; /* Flags for the fsync */ int szPage; /* Size of one page */ +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + void *aFrameBuf; /* Frame buffer */ + size_t szFrameBuf; /* Size of frame buffer */ +#endif } WalWriter; /* ** Write iAmt bytes of content into the WAL file beginning at iOffset. ** Do a sync when crossing the p->iSyncPoint boundary. @@ -3263,17 +3274,29 @@ int nTruncate, /* The commit flag. Usually 0. >0 for commit */ sqlite3_int64 iOffset /* Byte offset at which to write */ ){ int rc; /* Result code from subfunctions */ void *pData; /* Data actually written */ +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + u8 *aFrame = p->aFrameBuf; +#else u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ +#endif + pData = pPage->pData; + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + memcpy(&aFrame[WAL_FRAME_HDRSIZE], pData, p->szPage); + rc = walWriteToLog(p, aFrame, (p->szPage + WAL_FRAME_HDRSIZE), iOffset); +#else rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); if( rc ) return rc; /* Write the page data */ rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); +#endif return rc; } /* ** This function is called as part of committing a transaction within which @@ -3428,10 +3451,17 @@ w.iSyncPoint = 0; w.syncFlags = sync_flags; w.szPage = szPage; iOffset = walFrameOffset(iFrame+1, szPage); szFrame = szPage + WAL_FRAME_HDRSIZE; +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + w.aFrameBuf = (void *)malloc(szFrame); + if( NULL==w.aFrameBuf ){ + return SQLITE_NOMEM; + } +#endif + /* Write all frames into the log file exactly once */ for(p=pList; p; p=p->pDirty){ int nDbSize; /* 0 normally. Positive == commit flag */ @@ -3459,11 +3489,16 @@ iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; rc = walWriteOneFrame(&w, p, nDbSize, iOffset); - if( rc ) return rc; + if( rc ) { +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + free(w.aFrameBuf); +#endif + return rc; + } pLast = p; iOffset += szFrame; p->flags |= PGHDR_WAL_APPEND; } @@ -3494,11 +3529,16 @@ w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; bSync = (w.iSyncPoint==iOffset); testcase( bSync ); while( iOffsettruncateOnCommit && pWal->mxWalSize>=0 ){ Index: src/wal.h ================================================================== --- src/wal.h +++ src/wal.h @@ -53,11 +53,11 @@ ** There is one object of this type for each pager. */ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, int, Wal**); int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ void sqlite3WalLimit(Wal*, i64); @@ -76,10 +76,13 @@ int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); +/* If a WAL journal file has been created, return it */ +sqlite3_file *sqlite3WalFile(Wal *pWal); + /* Obtain or release the WRITER lock. */ int sqlite3WalBeginWriteTransaction(Wal *pWal); int sqlite3WalEndWriteTransaction(Wal *pWal); /* Undo any frames written (but not committed) to the log */ Index: test/8_3_names.test ================================================================== --- test/8_3_names.test +++ test/8_3_names.test @@ -144,10 +144,12 @@ # ifcapable !wal { finish_test return } +if ![wal_is_ok] { finish_test; return } + db close forcedelete test.db do_test 8_3_names-5.0 { sqlite3 db file:./test.db?8_3_names=1 load_static_extension db wholenumber Index: test/altertab.test ================================================================== --- test/altertab.test +++ test/altertab.test @@ -548,11 +548,11 @@ do_catchsql_test 16.20 { DROP TABLE y1_segments; } {1 {table y1_segments may not be dropped}} - do_catchsql_test 16.20 { + do_catchsql_test 16.21 { ALTER TABLE y1_segments RENAME TO abc; } {1 {table y1_segments may not be altered}} sqlite3_db_config db DEFENSIVE 0 do_catchsql_test 16.22 { ALTER TABLE y1_segments RENAME TO abc; @@ -568,10 +568,14 @@ do_catchsql_test 16.25 { ALTER TABLE abc RENAME TO y1_segments; } {0 {}} sqlite3_db_config db DEFENSIVE 1 + do_catchsql_test 16.21 { + DROP TABLE y1_segments; + } {1 {table y1_segments may not be dropped}} + do_execsql_test 16.30 { ALTER TABLE y1 RENAME TO z1; } do_execsql_test 16.40 { Index: test/attach.test ================================================================== --- test/attach.test +++ test/attach.test @@ -758,27 +758,28 @@ catchsql { ATTACH DATABASE 'no-such-file' AS nosuch; } } {0 {}} if {$tcl_platform(platform)=="unix"} { - do_test attach-6.2 { - sqlite3 dbx cannot-read - dbx eval {CREATE TABLE t1(a,b,c)} - dbx close - file attributes cannot-read -permission 0000 - if {[file writable cannot-read]} { - puts "\n**** Tests do not work when run as root ****" - forcedelete cannot-read - exit 1 - } - catchsql { - ATTACH DATABASE 'cannot-read' AS noread; - } - } {1 {unable to open database: cannot-read}} - do_test attach-6.2.2 { - db errorcode - } {14} + sqlite3 dbx cannot-read + dbx eval {CREATE TABLE t1(a,b,c)} + dbx close + file attributes cannot-read -permission 0000 + if {[file writable cannot-read]} { + #puts "\n**** Tests do not work when run as root ****" + forcedelete cannot-read + #exit 1 + } else { + do_test attach-6.2 { + catchsql { + ATTACH DATABASE 'cannot-read' AS noread; + } + } {1 {unable to open database: cannot-read}} + do_test attach-6.2.2 { + db errorcode + } {14} + } forcedelete cannot-read } # Check the error message if we try to access a database that has # not been attached. Index: test/attach4.test ================================================================== --- test/attach4.test +++ test/attach4.test @@ -78,10 +78,11 @@ set mode delete } else { set mode wal } ifcapable !wal { set mode delete } + if ![wal_is_ok] { set mode delete } lappend L $mode append S " PRAGMA $name.journal_mode = WAL; UPDATE $name.tbl SET x = '$name'; " Index: test/bind.test ================================================================== --- test/bind.test +++ test/bind.test @@ -654,10 +654,13 @@ sqlite3_step $VM list [sqlite3_column_type $VM 0] [sqlite3_column_type $VM 1] \ [sqlite3_column_type $VM 2] } {NULL NULL NULL} sqlite3_finalize $VM +do_test bind-13.5 { + sqlite3_clear_bindings_null +} {0} #-------------------------------------------------------------------- # These tests attempt to reproduce bug #3463. # proc param_names {db zSql} { Index: test/cache.test ================================================================== --- test/cache.test +++ test/cache.test @@ -105,16 +105,21 @@ INSERT INTO t2 VALUES(1, 2); PRAGMA lock_status; } {main exclusive temp closed} do_test cache-2.3.4 { pager_cache_size db } 2 do_execsql_test cache-2.3.5 COMMIT -do_test cache-2.3.6 { pager_cache_size db } 1 - +if !$::sqlite_options(enable_purgeable_pcache) { + # purgeable pcache doesn't share cache between connections + do_test cache-2.3.6 { pager_cache_size db } 1 +} do_execsql_test cache-2.3.7 { SELECT * FROM t1 UNION SELECT * FROM t2; } {1 2 i j x y} -do_test cache-2.3.8 { pager_cache_size db } 1 +if !$::sqlite_options(enable_purgeable_pcache) { + # purgeable pcache doesn't share cache between connections + do_test cache-2.3.8 { pager_cache_size db } 1 +} # Tests for cache_size = 0. # do_execsql_test cache-2.4.1 { PRAGMA cache_size = 0; @@ -128,13 +133,19 @@ PRAGMA lock_status; } {main exclusive temp closed} do_test cache-2.4.4 { pager_cache_size db } 2 do_execsql_test cache-2.4.5 COMMIT -do_test cache-2.4.6 { pager_cache_size db } 0 +if !$::sqlite_options(enable_purgeable_pcache) { + # purgeable pcache doesn't share cache between connections + do_test cache-2.4.6 { pager_cache_size db } 0 +} do_execsql_test cache-2.4.7 { SELECT * FROM t1 UNION SELECT * FROM t2; } {1 2 i j x y} -do_test cache-2.4.8 { pager_cache_size db } 0 +if !$::sqlite_options(enable_purgeable_pcache) { + # purgeable pcache doesn't share cache between connections + do_test cache-2.4.8 { pager_cache_size db } 0 +} sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit) finish_test Index: test/dbstatus2.test ================================================================== --- test/dbstatus2.test +++ test/dbstatus2.test @@ -40,10 +40,13 @@ proc db_spill {db {reset 0}} { sqlite3_db_status $db CACHE_SPILL $reset } do_test 1.1 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close sqlite3 db test.db execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 Index: test/e_wal.test ================================================================== --- test/e_wal.test +++ test/e_wal.test @@ -31,11 +31,12 @@ # 1.2: "read" tests. # 1.3: "write" tests. # # All three done with VFS "oldvfs", which has iVersion==1 and so does # not support shared memory. -# +# +forcedelete test.db-shm sqlite3 db test.db -vfs oldvfs do_execsql_test 1.1.1 { PRAGMA journal_mode = WAL; } {delete} do_execsql_test 1.1.2 { Index: test/e_walckpt.test ================================================================== --- test/e_walckpt.test +++ test/e_walckpt.test @@ -438,10 +438,11 @@ # in the SQLITE_CHECKPOINT_PASSIVE mode. # # It's not. Test case "$tp.6". # do_test $tp.4 { + forcedelete abc.db-shm abc.db-wal forcecopy test.db abc.db sqlite3 db4 abc.db db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } } {1 2 3 4 5 6} do_test $tp.5 { set ::sync_counter } 0 Index: test/exclusive.test ================================================================== --- test/exclusive.test +++ test/exclusive.test @@ -402,16 +402,12 @@ sqlite3 db test.db # if we're using proxy locks, we use 3 filedescriptors for a db # that is open but NOT writing changes, normally # sqlite uses 1 (proxy locking adds the conch and the local lock) -set using_proxy 0 -foreach {name value} [array get env SQLITE_FORCE_PROXY_LOCKING] { - set using_proxy $value -} set extrafds 0 -if {$using_proxy!=0} { +if {[forced_proxy_locking]} { set extrafds 2 } do_test exclusive-5.0 { execsql { Index: test/fallocate.test ================================================================== --- test/fallocate.test +++ test/fallocate.test @@ -86,17 +86,28 @@ # set skipwaltests [expr { [permutation]=="journaltest" || [permutation]=="inmemory_journal" }] ifcapable !wal { set skipwaltests 1 } +if {![wal_is_ok]} { set skipwaltests 1 } if {!$skipwaltests} { db close forcedelete test.db + ifcapable enable_persist_wal { + forcedelete test.db-journal + forcedelete test.db-wal + forcedelete test.db-shm + } + if {[forced_proxy_locking]} { + forcedelete .test.db-conch + } sqlite3 db test.db + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } file_control_chunksize_test db main [expr 32*1024] - do_test fallocate-2.1 { execsql { PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); Index: test/filter1.test ================================================================== --- test/filter1.test +++ test/filter1.test @@ -161,10 +161,30 @@ SELECT a, avg(c) FILTER (WHERE b!=1) AS h FROM t1 GROUP BY a ORDER BY avg(c); } {c 2.0 a 10.0 b 5.0} do_execsql_test 4.4 { SELECT a, avg(c) FILTER (WHERE b!=1) FROM t1 GROUP BY a ORDER BY 2 } {c 2.0 b 5.0 a 10.0} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(1, 3); +} + +do_execsql_test 5.1 { + SELECT count(*) FILTER (WHERE b>2) FROM (SELECT * FROM t1) +} {1} + +do_execsql_test 5.2 { + SELECT count(*) FILTER (WHERE b>2) OVER () FROM (SELECT * FROM t1) +} {1 1} + +do_execsql_test 5.3 { + SELECT count(*) FILTER (WHERE b>2) OVER (ORDER BY b) FROM (SELECT * FROM t1) +} {0 1} #------------------------------------------------------------------------- reset_db do_execsql_test 5.0 { CREATE TABLE t1(a, b); Index: test/incrblob3.test ================================================================== --- test/incrblob3.test +++ test/incrblob3.test @@ -258,14 +258,19 @@ hexio_write test.db 24 [hexio_render_int32 $dbversion] return "" } -do_test incrblob3-7.2 { - sqlite3 db test.db - sqlite3_db_config_lookaside db 0 0 0 - list [catch {db incrblob blobs v 1} msg] $msg -} {1 {database schema has changed}} -db close +#set sqlite_os_trace 1 +# AFP asserts because the "db incrblob blobs v 1" clears the file locks and the unlock fails (HFS doesn't care about a failed unlock) +if {[path_is_local "."]} { + + do_test incrblob3-7.2 { + sqlite3 db test.db + sqlite3_db_config_lookaside db 0 0 0 + list [catch {db incrblob blobs v 1} msg] $msg + } {1 {database schema has changed}} + db close +} tvfs delete finish_test Index: test/incrvacuum2.test ================================================================== --- test/incrvacuum2.test +++ test/incrvacuum2.test @@ -13,10 +13,11 @@ # # $Id: incrvacuum2.test,v 1.6 2009/07/25 13:42:50 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/wal_common.tcl # If this build of the library does not support auto-vacuum, omit this # whole file. ifcapable {!autovacuum || !pragma} { finish_test @@ -131,10 +132,12 @@ COMMIT; } } {} integrity_check incrvacuum2-3.3 + +if ![wal_is_ok] { finish_test; return } if {[wal_is_capable]} { # At one point, when a specific page was being extracted from the b-tree # free-list (e.g. during an incremental-vacuum), all trunk pages that # occurred before the specific page in the free-list trunk were being Index: test/join.test ================================================================== --- test/join.test +++ test/join.test @@ -881,10 +881,31 @@ } {1 1 1 1} do_execsql_test join-17.110 { SELECT * FROM t1 LEFT JOIN (SELECT abs(1)+2 AS y FROM t1) ON x WHERE NOT(y='a'); } {1 3 1 3} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test join-18.1 { + CREATE TABLE t0(a); + CREATE TABLE t1(b); + CREATE VIEW v0 AS SELECT a FROM t1 LEFT JOIN t0; + INSERT INTO t1 VALUES (1); +} {} + +do_execsql_test join-18.2 { + SELECT * FROM v0 WHERE NOT(v0.a IS FALSE); +} {{}} + +do_execsql_test join-18.3 { + SELECT * FROM t1 LEFT JOIN t0 WHERE NOT(a IS FALSE); +} {1 {}} + +do_execsql_test join-18.4 { + SELECT NOT(v0.a IS FALSE) FROM v0 +} {1} #------------------------------------------------------------------------- reset_db do_execsql_test join-18.1 { CREATE TABLE t0(a); Index: test/journal3.test ================================================================== --- test/journal3.test +++ test/journal3.test @@ -18,11 +18,11 @@ #------------------------------------------------------------------------- # If a connection is required to create a journal file, it creates it with # the same file-system permissions as the database file itself. Test this. # -if {$::tcl_platform(platform) == "unix" +if {$::tcl_platform(platform) == "unix" && ![path_is_dos "."] && [atomic_batch_write test.db]==0 } { # Changed on 2012-02-13: umask is deliberately ignored for -wal, -journal, # and -shm files. Index: test/lock5.test ================================================================== --- test/lock5.test +++ test/lock5.test @@ -104,92 +104,98 @@ if {[catch {sqlite3 db test.db -vfs unix-flock} msg]} { finish_test return } -do_test lock5-flock.1 { - sqlite3 db test.db -vfs unix-flock - execsql { - CREATE TABLE t1(a, b); - BEGIN; - INSERT INTO t1 VALUES(1, 2); - } -} {} - -# Make sure we are not accidentally using the dotfile locking scheme. -do_test lock5-flock.2 { - file exists test.db.lock -} {0} - -do_test lock5-flock.3 { - catch { sqlite3 db2 test.db -vfs unix-flock } - catchsql { SELECT * FROM t1 } db2 -} {1 {database is locked}} - -do_test lock5-flock.4 { - execsql COMMIT - catchsql { SELECT * FROM t1 } db2 -} {0 {1 2}} - -do_test lock5-flock.5 { - execsql BEGIN - catchsql { SELECT * FROM t1 } db2 -} {0 {1 2}} - -do_test lock5-flock.6 { - execsql {SELECT * FROM t1} - catchsql { SELECT * FROM t1 } db2 -} {1 {database is locked}} - -do_test lock5-flock.7 { - db close - catchsql { SELECT * FROM t1 } db2 -} {0 {1 2}} - -do_test lock5-flock.8 { - db2 close -} {} - -##################################################################### - -do_test lock5-none.1 { - sqlite3 db test.db -vfs unix-none - sqlite3 db2 test.db -vfs unix-none - execsql { PRAGMA mmap_size = 0 } db2 - execsql { - BEGIN; - INSERT INTO t1 VALUES(3, 4); - } -} {} -do_test lock5-none.2 { - execsql { SELECT * FROM t1 } -} {1 2 3 4} -do_test lock5-none.3 { - execsql { SELECT * FROM t1; } db2 -} {1 2} -do_test lock5-none.4 { - execsql { - BEGIN; - SELECT * FROM t1; - } db2 -} {1 2} -do_test lock5-none.5 { - execsql COMMIT - execsql {SELECT * FROM t1} db2 -} {1 2} - -ifcapable memorymanage { +# Only run the flock tests on a local file system +if { [path_is_local "."] } { + + do_test lock5-flock.1 { + sqlite3 db test.db -vfs unix-flock + execsql { + CREATE TABLE t1(a, b); + BEGIN; + INSERT INTO t1 VALUES(1, 2); + } + } {} + + # Make sure we are not accidentally using the dotfile locking scheme. + do_test lock5-flock.2 { + file exists test.db.lock + } {0} + + do_test lock5-flock.3 { + catch { sqlite3 db2 test.db -vfs unix-flock } + catchsql { SELECT * FROM t1 } db2 + } {1 {database is locked}} + + do_test lock5-flock.4 { + execsql COMMIT + catchsql { SELECT * FROM t1 } db2 + } {0 {1 2}} + + do_test lock5-flock.5 { + execsql BEGIN + catchsql { SELECT * FROM t1 } db2 + } {0 {1 2}} + + do_test lock5-flock.6 { + execsql {SELECT * FROM t1} + catchsql { SELECT * FROM t1 } db2 + } {1 {database is locked}} + + do_test lock5-flock.7 { + db close + catchsql { SELECT * FROM t1 } db2 + } {0 {1 2}} + + do_test lock5-flock.8 { + db2 close + } {} + + ##################################################################### + + do_test lock5-none.1 { + sqlite3 db test.db -vfs unix-none + sqlite3 db2 test.db -vfs unix-none + execsql { + BEGIN; + INSERT INTO t1 VALUES(3, 4); + } + } {} + do_test lock5-none.2 { + execsql { SELECT * FROM t1 } + } {1 2 3 4} + do_test lock5-flock.3 { + execsql { SELECT * FROM t1 } db2 + } {1 2} + do_test lock5-none.4 { + execsql { + BEGIN; + SELECT * FROM t1; + } db2 + } {1 2} + do_test lock5-none.5 { + execsql COMMIT + } {} do_test lock5-none.6 { sqlite3_release_memory 1000000 execsql {SELECT * FROM t1} db2 - } {1 2 3 4} -} + } {1 2} + + ifcapable memorymanage { + do_test lock5-none.6 { + sqlite3_release_memory 1000000 + execsql {SELECT * FROM t1} db2 + } {1 2 3 4} + } -do_test lock5-none.X { - db close - db2 close -} {} + do_test lock5-flock.X { + db close + db2 close + } {} +} ifcapable lock_proxy_pragmas { set env(SQLITE_FORCE_PROXY_LOCKING) $::using_proxy } Index: test/lock6.test ================================================================== --- test/lock6.test +++ test/lock6.test @@ -124,22 +124,23 @@ PRAGMA lock_proxy_file; } db] set lockpath } {{:auto: (not held)}} + set lpp [exec mktemp -t fail] do_test lock6-1.4.1 { + execsql "PRAGMA lock_proxy_file='$lpp'" catchsql { - PRAGMA lock_proxy_file="notmine"; select * from sqlite_master; } db } {1 {database is locked}} do_test lock6-1.4.2 { execsql { PRAGMA lock_proxy_file; } db - } {notmine} + } $lpp do_test lock6-1.5 { testfixture $::tf1 { db eval { BEGIN; @@ -148,13 +149,14 @@ } } {} catch {testfixture $::tf1 {db close}} + set lpp [exec mktemp -t ok] do_test lock6-1.6 { + execsql "PRAGMA lock_proxy_file='$lpp'" execsql { - PRAGMA lock_proxy_file="mine"; select * from sqlite_master; } db } {} catch {close $::tf1} ADDED test/lock_proxy.test Index: test/lock_proxy.test ================================================================== --- /dev/null +++ test/lock_proxy.test @@ -0,0 +1,142 @@ +# 2008 June 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is database proxy locks. +# +# $Id$ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# This file is only run if using the unix backend compiled with the +# SQLITE_ENABLE_LOCKING_STYLE macro. +db close +if {[catch {sqlite3 db test.db -vfs unix-none} msg]} { + finish_test + return +} +db close +file delete -force test.db.lock + +##################################################################### +ifcapable lock_proxy_pragmas&&prefer_proxy_locking { + set ::using_proxy 0 + foreach {name value} [array get env SQLITE_FORCE_PROXY_LOCKING] { + set ::using_proxy $value + } + + # enable the proxy locking for these tests + set env(SQLITE_FORCE_PROXY_LOCKING) "1" + + # test conch file creation + # + + catch { file delete lock_proxy_test.db } + catch { file delete .lock_proxy_test.db-conch } + # test that proxy locking mode creates conch files + do_test lock_proxy1.0 { + sqlite3 db2 lock_proxy_test.db + catchsql { + create table x(y); + } db2 + file exists .lock_proxy_test.db-conch + } {1} + catch { db2 close } + + + # test proxy locking readonly file system handling + # + + if {[file exists /usr/bin/hdiutil]} { + + puts "Creating readonly file system for proxy locking tests..." + if {[file exists /Volumes/readonly]} { + exec hdiutil detach /Volumes/readonly + } + if {[file exists readonly.dmg]} { + file delete readonly.dmg + } + exec hdiutil create -megabytes 1 -fs HFS+ readonly.dmg -volname readonly + exec hdiutil attach readonly.dmg + + # create test1.db and a .test1.db-conch for host4 + set sqlite_hostid_num 4 + sqlite3 db2 /Volumes/readonly/test1.db + execsql { + create table x(y); + } db2 + db2 close + + # create test2.db and a .test2.db-conch for host5 + set sqlite_hostid_num 5 + sqlite3 db2 /Volumes/readonly/test2.db + execsql { + create table x(y); + } db2 + db2 close + + # create test3.db without a conch file + set env(SQLITE_FORCE_PROXY_LOCKING) "0" + sqlite3 db2 /Volumes/readonly/test3.db + execsql { + create table x(y); + } db2 + db2 close + exec hdiutil detach /Volumes/readonly + exec hdiutil attach -readonly readonly.dmg + + # test that an unwritable, host-mismatched conch file prevents + # read only proxy-locking mode database access + set env(SQLITE_FORCE_PROXY_LOCKING) "1" + do_test lock_proxy2.0 { + sqlite3 db2 /Volumes/readonly/test1.db + catchsql { + select * from sqlite_master; + } db2 + } {1 {database is locked}} + catch { db2 close } + + # test that an unwritable, host-matching conch file allows + # read only proxy-locking mode database access + do_test lock_proxy2.1 { + sqlite3 db2 /Volumes/readonly/test2.db + catchsql { + select * from sqlite_master; + } db2 + } {0 {table x x 2 {CREATE TABLE x(y)}}} + catch { db2 close } + + # test that an unwritable, nonexistant conch file allows + # read only proxy-locking mode database access + do_test lock_proxy2.2 { + sqlite3 db2 /Volumes/readonly/test3.db + catchsql { + select * from sqlite_master; + } db2 + } {0 {table x x 2 {CREATE TABLE x(y)}}} + catch { db2 close } + + exec hdiutil detach /Volumes/readonly + file delete readonly.dmg + } + set env(SQLITE_FORCE_PROXY_LOCKING) "0" + set sqlite_hostid_num 0 +} + +##################################################################### + +file delete -force test.db + +ifcapable lock_proxy_pragmas&&prefer_proxy_locking { + set env(SQLITE_FORCE_PROXY_LOCKING) $::using_proxy +} + +finish_test Index: test/main.test ================================================================== --- test/main.test +++ test/main.test @@ -312,19 +312,19 @@ # Here are some tests for tokenize.c. # do_test main-3.1 { catch {db close} - foreach f [glob -nocomplain testdb/*] {forcedelete $f} + catch {foreach f [glob -nocomplain testdb/*] {forcedelete $f}} forcedelete testdb sqlite3 db testdb set v [catch {execsql {SELECT * from T1 where x!!5}} msg] lappend v $msg } {1 {unrecognized token: "!"}} do_test main-3.2 { catch {db close} - foreach f [glob -nocomplain testdb/*] {forcedelete $f} + catch {foreach f [glob -nocomplain testdb/*] {forcedelete $f}} forcedelete testdb sqlite3 db testdb set v [catch {execsql {SELECT * from T1 where ^x}} msg] lappend v $msg } {1 {unrecognized token: "^"}} @@ -440,11 +440,11 @@ } {0 123} do_test main-3.3 { catch {db close} - foreach f [glob -nocomplain testdb/*] {forcedelete $f} + catch {foreach f [glob -nocomplain testdb/*] {forcedelete $f}} forcedelete testdb sqlite3 db testdb execsql { create table T1(X REAL); /* C-style comments allowed */ insert into T1 values(0.5); Index: test/malloc_common.tcl ================================================================== --- test/malloc_common.tcl +++ test/malloc_common.tcl @@ -443,11 +443,11 @@ # failing. # set zRepeat "transient" if {$::iRepeat} {set zRepeat "persistent"} restore_prng_state - foreach file [glob -nocomplain test.db-mj*] {forcedelete $file} + catch {foreach file [glob -nocomplain test.db-mj*] {forcedelete $file}} do_test ${tn}.${zRepeat}.${::n} { # Remove all traces of database files test.db and test2.db # from the file-system. Then open (empty database) "test.db" Index: test/manydb.test ================================================================== --- test/manydb.test +++ test/manydb.test @@ -20,16 +20,12 @@ set N 300 # if we're using proxy locks, we use 5 filedescriptors for a db # that is open and in the middle of writing changes, normally # sqlite uses 3 (proxy locking adds the conch and the local lock) -set using_proxy 0 -foreach {name value} [array get env SQLITE_FORCE_PROXY_LOCKING] { - set using_proxy value -} set num_fd_per_openwrite_db 3 -if {$using_proxy>0} { +if {[forced_proxy_locking]} { set num_fd_per_openwrite_db 5 } # First test how many file descriptors are available for use. To open a # database for writing SQLite requires 3 file descriptors (the database, the Index: test/memdb.test ================================================================== --- test/memdb.test +++ test/memdb.test @@ -9,11 +9,10 @@ # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this script is in-memory database backend. # -# $Id: memdb.test,v 1.19 2009/05/18 16:04:38 danielk1977 Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -405,27 +404,30 @@ } } 0 # Test that auto-vacuum works with in-memory databases. # -ifcapable autovacuum { - do_test memdb-9.1 { - db close - sqlite3 db test.db - db cache size 0 - execsql { - PRAGMA auto_vacuum = full; - CREATE TABLE t1(a); - INSERT INTO t1 VALUES(randstr(1000,1000)); - INSERT INTO t1 VALUES(randstr(1000,1000)); - INSERT INTO t1 VALUES(randstr(1000,1000)); - } - set before [db one {PRAGMA page_count}] - execsql { DELETE FROM t1 } - set after [db one {PRAGMA page_count}] - expr {$before>$after} - } {1} +set msize [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] +if {[lindex $msize 2]!=0} { + ifcapable autovacuum { + do_test memdb-9.1 { + db close + sqlite3 db test.db + db cache size 0 + execsql { + PRAGMA auto_vacuum = full; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(randstr(1000,1000)); + INSERT INTO t1 VALUES(randstr(1000,1000)); + INSERT INTO t1 VALUES(randstr(1000,1000)); + } + set before [db one {PRAGMA page_count}] + execsql { DELETE FROM t1 } + set after [db one {PRAGMA page_count}] + expr {$before>$after} + } {1} + } } } ;# ifcapable memorydb finish_test Index: test/memsubsys1.test ================================================================== --- test/memsubsys1.test +++ test/memsubsys1.test @@ -122,22 +122,24 @@ sqlite3_config_pagecache [expr 512+$xtra_size] 20 sqlite3_config singlethread sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-3.1 {PRAGMA page_size=1024} -do_test memsubsys1-3.1.3 { - set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} 0 -do_test memsubsys1-3.1.4 { - set overflow [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] - # Note: The measured PAGECACHE_OVERFLOW is amount malloc() returns, not what - # was requested. System malloc() implementations might (arbitrarily) return - # slightly different oversize buffers, which can result in slightly different - # PAGECACHE_OVERFLOW sizes between consecutive runs. So we cannot do an - # exact comparison. Simply verify that the amount is within 5%. - expr {$overflow>=$max_pagecache*0.95 && $overflow<=$max_pagecache*1.05} -} 1 +if !$::sqlite_options(enable_purgeable_pcache) { + do_test memsubsys1-3.1.3 { + set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] + } 0 + do_test memsubsys1-3.1.4 { + set overflow [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] + # Note: The measured PAGECACHE_OVERFLOW is amount malloc() returns, not what + # was requested. System malloc() implementations might (arbitrarily) return + # slightly different oversize buffers, which can result in different + # PAGECACHE_OVERFLOW sizes between consecutive runs. So we cannot do an + # exact comparison. Simply verify that the amount is within 5%. + expr {$overflow>=$max_pagecache*0.95 && $overflow<=$max_pagecache*1.05} + } 1 +} do_test memsubsys1-3.1.5 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 0 db close sqlite3_shutdown @@ -167,13 +169,15 @@ #show_memstats do_test memsubsys1-4.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] expr {$pg_used>=45 && $pg_used<=50} } 1 -do_test memsubsys1-4.4 { - set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] -} 0 +if !$::sqlite_options(enable_purgeable_pcache) { + do_test memsubsys1-4.4 { + set pg_ovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] + } 0 +} do_test memsubsys1-4.5 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] expr {$maxreq<7000} } 1 Index: test/multiplex.test ================================================================== --- test/multiplex.test +++ test/multiplex.test @@ -11,10 +11,16 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl + +# AFP doesn't like multiplex db tests +if { ![path_is_local "."] } { + finish_test + return +} # The tests in this file assume that SQLite is compiled without # ENABLE_8_3_NAMES. # ifcapable 8_3_names { Index: test/pager1.test ================================================================== --- test/pager1.test +++ test/pager1.test @@ -551,10 +551,12 @@ incr ::mj_delete_cnt } return SQLITE_OK } +if {![forced_proxy_locking]} { + # proxy locking uses can't deal with auto proxy file paths longer than MAXPATHLEN foreach {tn1 tcl} { 1 { set prefix "test.db" } 2 { # This test depends on the underlying VFS being able to open paths # 512 bytes in length. The idea is to create a hot-journal file that @@ -737,10 +739,11 @@ cd $pwd } db close tv delete forcedelete $dirname +} # Set up a VFS to make a copy of the file-system just before deleting a # journal file to commit a transaction. The transaction modifies exactly # two database pages (and page 1 - the change counter). # @@ -1218,11 +1221,11 @@ # $js: The expected size of the journal file, in bytes, after executing # the SQL script. Or -1 if the journal is not expected to exist. # $ws: The expected size of the WAL file, in bytes, after executing # the SQL script. Or -1 if the WAL is not expected to exist. # -ifcapable wal { +if {$::sqlite_options(wal) && [wal_is_ok]} { faultsim_delete_and_reopen foreach {tn sql res js ws} [subst { 1 { CREATE TABLE t1(a, b); @@ -1254,10 +1257,13 @@ 8 { PRAGMA journal_mode = TRUNCATE } truncate 0 -1 9 { INSERT INTO t1 VALUES(7, 8) } {} 0 -1 10 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8} 0 -1 }] { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } do_execsql_test pager1-7.1.$tn.1 $sql $res catch { set J -1 ; set J [file size test.db-journal] } catch { set W -1 ; set W [file size test.db-wal] } do_test pager1-7.1.$tn.2 { list $J $W } [list $js $ws] } @@ -2046,11 +2052,11 @@ BEGIN EXCLUSIVE; COMMIT; } } {} -ifcapable wal { +if {$::sqlite_options(wal) && [wal_is_ok]} { do_test pager1-20.3.1 { faultsim_delete_and_reopen db func a_string a_string execsql { PRAGMA cache_size = 10; @@ -2081,11 +2087,11 @@ # Test that a WAL database may not be opened if: # # pager1-21.1.*: The VFS has an iVersion less than 2, or # pager1-21.2.*: The VFS does not provide xShmXXX() methods. # -ifcapable wal { +if {$::sqlite_options(wal) && [wal_is_ok]} { do_test pager1-21.0 { faultsim_delete_and_reopen execsql { PRAGMA journal_mode = WAL; CREATE TABLE ko(c DEFAULT 'abc', b DEFAULT 'def'); @@ -2112,37 +2118,40 @@ # Test that a "PRAGMA wal_checkpoint": # # pager1-22.1.*: is a no-op on a non-WAL db, and # pager1-22.2.*: does not cause xSync calls with a synchronous=off db. # -ifcapable wal { - do_test pager1-22.1.1 { - faultsim_delete_and_reopen - execsql { - CREATE TABLE ko(c DEFAULT 'abc', b DEFAULT 'def'); - INSERT INTO ko DEFAULT VALUES; - } - execsql { PRAGMA wal_checkpoint } - } {0 -1 -1} - do_test pager1-22.2.1 { - testvfs tv -default 1 - tv filter xSync - tv script xSyncCb - proc xSyncCb {args} {incr ::synccount} - set ::synccount 0 - sqlite3 db test.db - execsql { - PRAGMA synchronous = off; - PRAGMA journal_mode = WAL; - INSERT INTO ko DEFAULT VALUES; - } - execsql { PRAGMA wal_checkpoint } - set synccount - } {0} - db close - tv delete -} +do_test pager1-22.1.1 { + faultsim_delete_and_reopen + execsql { + CREATE TABLE ko(c DEFAULT 'abc', b DEFAULT 'def'); + INSERT INTO ko DEFAULT VALUES; + } + execsql { PRAGMA wal_checkpoint } +} {0 -1 -1} +do_test pager1-22.2.1 { + testvfs tv -default 1 + tv filter xSync + tv script xSyncCb + proc xSyncCb {args} {incr ::synccount} + sqlite3 db test.db + + # Switch the db to WAL mode. And then execute a SELECT to make sure + # that the WAL file is open. Note that this may change the synchronous + # setting if DEFAULT_WAL_SAFETYLEVEL is defined. + execsql { PRAGMA journal_mode = WAL ; SELECT * FROM ko } + + # Set synchronous=OFF. Insert some data and run a checkpoint. Since + # sync=off, this should not cause any calls to the xSync() method. + set ::synccount 0 + execsql { + PRAGMA synchronous = off; + INSERT INTO ko DEFAULT VALUES; + PRAGMA wal_checkpoint; + } + set synccount +} {0} #------------------------------------------------------------------------- # Tests for changing journal mode. # # pager1-23.1.*: Test that when changing from PERSIST to DELETE mode, @@ -2386,11 +2395,11 @@ # Test that attempting to open a write-transaction with # locking_mode=exclusive in WAL mode fails if there are other clients on # the same database. # catch { db close } -ifcapable wal { +if {$::sqlite_options(wal) && [wal_is_ok]} { do_multiclient_test tn { do_test pager1-28.$tn.1 { sql1 { PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); Index: test/pagerfault.test ================================================================== --- test/pagerfault.test +++ test/pagerfault.test @@ -1059,10 +1059,43 @@ } {ok} db close } } + +#------------------------------------------------------------------------- +# When a 3.7.0 client opens a write-transaction on a database file that +# has been appended to or truncated by a pre-370 client, it updates +# the db-size in the file header immediately. This test case provokes +# errors during that operation. +# +do_test pagerfault-22-pre1 { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(a_string(3000)); + CREATE TABLE t2(a); + INSERT INTO t2 VALUES(1); + } + db close + sql36231 { INSERT INTO t1 VALUES(a_string(3000)) } + faultsim_save_and_close +} {} +do_faultsim_test pagerfault-22 -prep { + faultsim_restore_and_reopen +} -body { + execsql { INSERT INTO t2 VALUES(2) } + execsql { SELECT * FROM t2 } +} -test { + faultsim_test_result {0 {1 2}} + faultsim_integrity_check +} + #------------------------------------------------------------------------- # When a 3.7.0 client opens a write-transaction on a database file that # has been appended to or truncated by a pre-370 client, it updates # the db-size in the file header immediately. This test case provokes Index: test/pcache.test ================================================================== --- test/pcache.test +++ test/pcache.test @@ -115,11 +115,13 @@ # Rolling back the transaction held by db2 at this point releases a pinned # page. Because the number of allocated pages is greater than the # configured maximum, this page should be freed immediately instead of # recycled. -# +# Note- purgable_pcache doesn't share caches between connections and these tests +# are only useful for testing that feature. +if !$::sqlite_options(enable_purgeable_pcache) { do_test pcache-1.8 { execsql {ROLLBACK} db2 pcache_stats } {current 23 max 22 min 20 recyclable 0} @@ -176,7 +178,10 @@ SELECT * FROM t8 ORDER BY a; SELECT * FROM t8; SELECT * FROM t9 ORDER BY a; SELECT * FROM t9; } pcache_stats } {current 14 max 15 min 10 recyclable 14} +} else { + db2 close +} finish_test Index: test/pragma.test ================================================================== --- test/pragma.test +++ test/pragma.test @@ -1528,11 +1528,16 @@ # Reset the sqlite3_temp_directory variable for the next run of tests: sqlite3 dbX :memory: dbX eval {PRAGMA temp_store_directory = ""} dbX close -ifcapable lock_proxy_pragmas&&prefer_proxy_locking { +set skip_lock_proxy_tests [path_is_dos "."] +ifcapable !(lock_proxy_pragmas&&prefer_proxy_locking) { + set skip_lock_proxy_tests 1 +} + +if !$skip_lock_proxy_tests { set sqlite_hostid_num 1 set using_proxy 0 foreach {name value} [array get env SQLITE_FORCE_PROXY_LOCKING] { set using_proxy $value @@ -1542,71 +1547,71 @@ # db close set env(SQLITE_FORCE_PROXY_LOCKING) "0" sqlite3 db test.db + # set lock proxy name and then query it via pragma interface + set lpp [exec mktemp -t "proxy1"] do_test pragma-16.1 { - execsql { - PRAGMA lock_proxy_file="mylittleproxy"; - select * from sqlite_master; - } - execsql { - PRAGMA lock_proxy_file; - } - } {mylittleproxy} - + execsql "PRAGMA lock_proxy_file='$lpp'" + execsql "select * from sqlite_master" + execsql "PRAGMA lock_proxy_file" + } $lpp + + # 2 database connections can share a lock proxy file do_test pragma-16.2 { sqlite3 db2 test.db - execsql { - PRAGMA lock_proxy_file="mylittleproxy"; - } db2 + execsql "PRAGMA lock_proxy_file='$lpp'" db2 } {} db2 close + # 2nd database connection should auto-name an existing lock proxy file do_test pragma-16.2.1 { sqlite3 db2 test.db execsql { PRAGMA lock_proxy_file=":auto:"; select * from sqlite_master; } db2 - execsql { - PRAGMA lock_proxy_file; - } db2 - } {mylittleproxy} + execsql "PRAGMA lock_proxy_file" db2 + } $lpp db2 close + set lpp2 [exec mktemp -t "proxy2"] + + # 2nd database connection cannot override the lock proxy file do_test pragma-16.3 { sqlite3 db2 test.db - execsql { - PRAGMA lock_proxy_file="myotherproxy"; - } db2 + execsql "PRAGMA lock_proxy_file='$lpp2'" db2 catchsql { select * from sqlite_master; } db2 } {1 {database is locked}} + set lpp3 [exec mktemp -t "proxy3"] + + # lock proxy file can be renamed if no other connections are active do_test pragma-16.4 { db2 close db close sqlite3 db2 test.db - execsql { - PRAGMA lock_proxy_file="myoriginalproxy"; - PRAGMA lock_proxy_file="myotherproxy"; - PRAGMA lock_proxy_file; - } db2 - } {myotherproxy} + execsql "PRAGMA lock_proxy_file='$lpp3'" db2 + execsql "PRAGMA lock_proxy_file='$lpp2'" db2 + execsql "PRAGMA lock_proxy_file" db2 + } $lpp2 db2 close set env(SQLITE_FORCE_PROXY_LOCKING) "1" + # auto-naming should reuse the last proxy name when available do_test pragma-16.5 { sqlite3 db2 test.db execsql { PRAGMA lock_proxy_file=":auto:"; PRAGMA lock_proxy_file; } db2 - } {myotherproxy} + } $lpp2 + # auto-naming a new proxy should use a predictable & unique name do_test pragma-16.6 { db2 close sqlite3 db2 test2.db set lockpath [execsql { PRAGMA lock_proxy_file=":auto:"; @@ -1614,10 +1619,11 @@ } db2] string match "*test2.db:auto:" $lockpath } {1} set sqlite_hostid_num 2 + # db access should be limited to one host at a time (simulate 2nd host id) do_test pragma-16.7 { list [catch { sqlite3 db test2.db execsql { PRAGMA lock_proxy_file=":auto:"; @@ -1625,39 +1631,138 @@ } } msg] $msg } {1 {database is locked}} db close + # default to using proxy locking (simulate network file system detection) do_test pragma-16.8 { list [catch { sqlite3 db test2.db execsql { select * from sqlite_master } } msg] $msg } {1 {database is locked}} db2 close + set lpp4 [exec mktemp -t "proxy4"] + + # check that db is unlocked after first host connection closes do_test pragma-16.8.1 { - execsql { - PRAGMA lock_proxy_file="yetanotherproxy"; - PRAGMA lock_proxy_file; - } - } {yetanotherproxy} + execsql "PRAGMA lock_proxy_file='$lpp4'" + execsql "select * from sqlite_master" + execsql "PRAGMA lock_proxy_file" + } $lpp4 + do_test pragma-16.8.2 { execsql { - create table mine(x); + create table if not exists mine(x); + insert into mine values (1); } } {} db close + file delete -force proxytest.db + file delete -force .proxytest.db-conch do_test pragma-16.9 { sqlite3 db proxytest.db set lockpath2 [execsql { PRAGMA lock_proxy_file=":auto:"; PRAGMA lock_proxy_file; } db] string match "*proxytest.db:auto:" $lockpath2 } {1} + + # ensure creating directories for a lock proxy file works + set lpp5d [exec mktemp -d -t "proxy5"] + set lpp5 $lpp5d/sub/dir/lock + db close + do_test pragma-16.10.1 { + sqlite3 db proxytest.db + execsql "PRAGMA lock_proxy_file='$lpp5'" + set lockpath2 [execsql { + PRAGMA lock_proxy_file; + } db] + string match "*sub/dir/lock" $lockpath2 + } {1} + + # ensure that after deleting the path, setting ":auto:" works correctly + db close + file delete -force $lpp5d + do_test pragma-16.10.2 { + sqlite3 db proxytest.db + set lockpath3 [execsql { + PRAGMA lock_proxy_file=":auto:"; + create table if not exists pt(y); + PRAGMA lock_proxy_file; + } db] + string match "*sub/dir/lock" $lockpath3 + } {1} + + # ensure that if the path can not be created (file instead of dir) + # setting :auto: deals with it by creating a new autonamed lock file + db close + file delete -force $lpp5d + close [open "$lpp5d" a] + do_test pragma-16.10.3 { + sqlite3 db proxytest.db + set lockpath2 [execsql { + PRAGMA lock_proxy_file=":auto:"; + create table if not exists zz(y); + PRAGMA lock_proxy_file; + } db] + string match "*proxytest.db:auto:" $lockpath2 + } {1} + + # make sure we can deal with ugly file paths correctly + db close + file delete -force $lpp5d + set lpp6 [exec mktemp -d -t "proxy6"]/./././////./proxytest/../proxytest/sub/dir/lock + do_test pragma-16.10.4 { + sqlite3 db proxytest.db + execsql "PRAGMA lock_proxy_file='$lpp6'" + set lockpath4 [execsql { + create table if not exists aa(bb); + PRAGMA lock_proxy_file; + } db] + string match "*proxytest/sub/dir/lock" $lockpath4 + } {1} + + # ensure that if the path can not be created (perm), setting :auto: deals + db close + file delete -force $lpp5d + do_test pragma-16.10.5 { + sqlite3 db proxytest.db + execsql "PRAGMA lock_proxy_file='$lpp5'" + execsql { + create table if not exists bb(bb); + } + db close + file delete -force $lpp5d + file mkdir $lpp5d + file attributes $lpp5d -permission 0000 + sqlite3 db proxytest.db + set lockpath5 [execsql { + PRAGMA lock_proxy_file=":auto:"; + create table if not exists cc(bb); + PRAGMA lock_proxy_file; + } db] + string match "*proxytest.db:auto:" $lockpath5 + } {1} + + # ensure that if the path can not be created, locking fails + db close + do_test pragma-16.10.6 { + sqlite3 db proxytest.db + execsql "PRAGMA lock_proxy_file='$lpp5'" + catchsql { + create table if not exists faily(y); + PRAGMA lock_proxy_file; + } db + } {1 {database is locked}} + db close + + file attributes $lpp5d -permission 0777 + file delete -force $lpp5d set env(SQLITE_FORCE_PROXY_LOCKING) $using_proxy set sqlite_hostid_num 0 } Index: test/pragma4.test ================================================================== --- test/pragma4.test +++ test/pragma4.test @@ -21,11 +21,11 @@ uplevel [list do_test $tn { sqlite3_column_count $::stmt } $nCol] sqlite3_finalize $::stmt } # If there is no RHS argument, the following PRAGMA statements operate as -# queries, returning a single row containing a single column. +# queries, returning a single row containing a single column. # # Or, if there is RHS argument, they return zero rows of zero columns. # foreach {tn sql} { 1 "PRAGMA application_id = 10" @@ -51,19 +51,21 @@ 22 "PRAGMA recursive_triggers = false" 23 "PRAGMA reverse_unordered_selects = false" 24 "PRAGMA schema_version = 211" 25 "PRAGMA short_column_names = 1" 26 "PRAGMA synchronous = full" + 27 "PRAGMA temp_store_directory = '/tmp'" + 28 "PRAGMA temp_store_directory = ''" 29 "PRAGMA temp_store = memory" 30 "PRAGMA user_version = 405" 31 "PRAGMA writable_schema = 1" } { reset_db # Without RHS: do_pragma_ncol_test 1.$tn.1 [lindex [split $sql =] 0] 1 - + # With RHS: do_pragma_ncol_test 1.$tn.2 $sql 0 } # These pragmas should never return any values. @@ -74,11 +76,11 @@ 3 "PRAGMA case_sensitive_like = 0" 4 "PRAGMA case_sensitive_like = 1" 5 "PRAGMA case_sensitive_like" } { - do_pragma_ncol_test 1.$tn.1 $sql 0 + do_pragma_ncol_test 2.$tn.1 $sql 0 } # EXPLAIN on a PRAGMA integrity_check. # Verify that that P4_INTARRAY argument to OP_IntegrityCk is rendered # correctly. Index: test/rowallock.test ================================================================== --- test/rowallock.test +++ test/rowallock.test @@ -9,15 +9,21 @@ # May you share freely, never taking more than you give. # #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing locks on read-only WAL-mode databases. +# +# Note that the apple-osx branch does not allow read-only WAL-mode +# databases (per check-in from Adam Swift on 2011-06-24 20:47:06) and +# so this test module is disabled in the apple-osx branch. set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl set testprefix rowallock + +finish_test; return ;# This test module disabled in the apple-osx branch set mmap_res 1000000 ifcapable !mmap { set mmap_res 0 } Index: test/shared.test ================================================================== --- test/shared.test +++ test/shared.test @@ -43,17 +43,13 @@ } # if we're using proxy locks, we use 2 filedescriptors for a db # that is open but NOT yet locked, after a lock is taken we'll have 3, # normally sqlite uses 1 (proxy locking adds the conch and the local lock) -set using_proxy 0 -foreach {name value} [array get env SQLITE_FORCE_PROXY_LOCKING] { - set using_proxy $value -} set extrafds_prelock 0 set extrafds_postlock 0 -if {$using_proxy>0} { +if {[forced_proxy_locking]} { set extrafds_prelock 1 set extrafds_postlock 2 } # $av is currently 0 if this loop iteration is to test with auto-vacuum turned Index: test/shrink.test ================================================================== --- test/shrink.test +++ test/shrink.test @@ -15,10 +15,15 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl test_set_config_pagecache 0 0 +# purgeable pcache tracks memory differently +ifcapable {enable_purgeable_pcache} { + finish_test + return +} unset -nocomplain baseline do_test shrink-1.1 { db eval { PRAGMA cache_size = 2000; CREATE TABLE t1(x,y); Index: test/stmt.test ================================================================== --- test/stmt.test +++ test/stmt.test @@ -33,72 +33,89 @@ # if {$::TEMP_STORE==3} { finish_test return } + +# if we're using proxy locks, we use 3 filedescriptors for a db +# that is open but NOT writing changes, normally +# sqlite uses 1 (proxy locking adds the conch and the local lock) +set extrafds 0 +if {[forced_proxy_locking]} { + set extrafds 2 +} + + do_test stmt-1.2 { set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds } {1} do_test stmt-1.3 { execsql { PRAGMA temp_store = file; BEGIN; INSERT INTO t1 VALUES(1, 1); } set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds } {2} do_test stmt-1.4 { execsql { INSERT INTO t1 SELECT a+1, b+1 FROM t1; } set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds # 2016-03-04: statement-journal open deferred } {2} do_test stmt-1.5 { execsql COMMIT set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds } {1} do_test stmt-1.6.1 { execsql { BEGIN; INSERT INTO t1 SELECT a+2, b+2 FROM t1; } set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds } {2} do_test stmt-1.6.2 { execsql { INSERT INTO t1 SELECT a+4, b+4 FROM t1 } set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds # 2016-03-04: statement-journal open deferred } {2} do_test stmt-1.7 { execsql COMMIT set sqlite_open_file_count + expr $sqlite_open_file_count-$extrafds } {1} -proc filecount {testname sql expected} { +proc filecount {testname sql expected extrafds} { uplevel [list do_test $testname [subst -nocommand { execsql BEGIN execsql { $sql } set ret [set sqlite_open_file_count] execsql ROLLBACK set ret - }] $expected] + }] [ expr $expected+$extrafds ] ] } -filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9) } 2 -filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2 -filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9 } 2 +filecount stmt-2.1 { INSERT INTO t1 VALUES(9, 9) } 2 $extrafds +filecount stmt-2.2 { REPLACE INTO t1 VALUES(9, 9) } 2 $extrafds +filecount stmt-2.3 { INSERT INTO t1 SELECT 9, 9 } 2 $extrafds filecount stmt-2.4 { INSERT INTO t1 SELECT 9, 9; INSERT INTO t1 SELECT 10, 10; -} 2 +} 2 $extrafds do_test stmt-2.5 { execsql { CREATE INDEX i1 ON t1(b) } } {} filecount stmt-2.6 { REPLACE INTO t1 VALUES(5, 5); REPLACE INTO t1 VALUES(5, 5); -} 2 +} 2 $extrafds finish_test Index: test/superlock.test ================================================================== --- test/superlock.test +++ test/superlock.test @@ -11,10 +11,11 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl +source $testdir/wal_common.tcl set testprefix superlock do_not_use_codec # Test organization: @@ -40,10 +41,15 @@ # 6.*: Test that if a superlocked WAL database is overwritten, existing # clients run the recovery to build the new wal-index after the # superlock is released. # # + +if {[forced_proxy_locking]} { + finish_test + return +} do_execsql_test 1.1 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); PRAGMA journal_mode = DELETE; @@ -55,10 +61,16 @@ } do_test 1.2 { sqlite3demo_superlock unlock test.db } {unlock} do_catchsql_test 1.3 { SELECT * FROM t1 } {1 {database is locked}} do_test 1.4 { unlock } {} + +ifcapable !wal {finish_test ; return } +if { ![wal_is_ok] } { + finish_test + return +} do_execsql_test 2.1 { INSERT INTO t1 VALUES(3, 4); PRAGMA journal_mode = WAL; } {wal} Index: test/tempdb.test ================================================================== --- test/tempdb.test +++ test/tempdb.test @@ -54,10 +54,15 @@ do_test tempdb-1.2 { execsql { SELECT * FROM t1 } } {} + +set extrafds 0 +if {[forced_proxy_locking]} { + set extrafds 2 +} do_test tempdb-2.1 { # Set $::jrnl_in_memory if the journal file is expected to be in-memory. # Similarly, set $::subj_in_memory if the sub-journal file is expected # to be in memory. These variables are used to calculate the expected @@ -79,11 +84,11 @@ INSERT INTO t2 VALUES(7, 8, 9); INSERT INTO t2 SELECT * FROM t1; } catchsql { INSERT INTO t1 SELECT * FROM t2 } set sqlite_open_file_count -} [expr 1 + (0==$jrnl_in_memory)] +} [expr 1 + $extrafds + (0==$jrnl_in_memory)] do_test tempdb-2.3 { execsql { PRAGMA temp_store = 'memory'; ROLLBACK; BEGIN; @@ -91,8 +96,8 @@ INSERT INTO t1 VALUES(4, 5, 6); INSERT INTO t2 SELECT * FROM t1; } catchsql { INSERT INTO t1 SELECT * FROM t2 } set sqlite_open_file_count -} [expr 1 + (0==$jrnl_in_memory)] +} [expr 1 + $extrafds + (0==$jrnl_in_memory)] finish_test Index: test/tester.tcl ================================================================== --- test/tester.tcl +++ test/tester.tcl @@ -592,18 +592,43 @@ # way if an individual test file changes the soft-heap-limit, it # will be reset at the start of the next test file. # sqlite3_soft_heap_limit64 $cmdlinearg(soft-heap-limit) sqlite3_hard_heap_limit64 $cmdlinearg(hard-heap-limit) + +proc forced_proxy_locking {} { + if $::sqlite_options(lock_proxy_pragmas)&&$::sqlite_options(prefer_proxy_locking) { + set force_proxy_value 0 + set force_key "SQLITE_FORCE_PROXY_LOCKING=" + foreach {env_pair} [exec env] { + if { [string first $force_key $env_pair] == 0} { + set force_proxy_value [string range $env_pair [string length $force_key] end] + } + } + if { "$force_proxy_value " == "1 " } { + return 1 + } + } + return 0 +} + # Create a test database # proc reset_db {} { catch {db close} forcedelete test.db forcedelete test.db-journal forcedelete test.db-wal + if {[forced_proxy_locking]} { + sqlite3 db ./test.db + set lock_proxy_path [db eval "PRAGMA lock_proxy_file;"] + catch {db close} + # puts "deleting $lock_proxy_path" + file delete -force $lock_proxy_path + file delete -force test.db + } sqlite3 db ./test.db set ::DB [sqlite3_connection_pointer db] if {[info exists ::SETUP_SQL]} { db eval $::SETUP_SQL } @@ -1312,15 +1337,19 @@ } } if {[info commands vdbe_coverage]!=""} { vdbe_coverage_report } - foreach f [glob -nocomplain test.db-*-journal] { - forcedelete $f + catch { + foreach f [glob -nocomplain test.db-*-journal] { + forcedelete $f + } } - foreach f [glob -nocomplain test.db-mj*] { - forcedelete $f + catch { + foreach f [glob -nocomplain test.db-mj*] { + forcedelete $f + } } exit [expr {$nErr>0}] } proc vdbe_coverage_report {} { @@ -2227,10 +2256,23 @@ proc presql {} { set presql "" catch {set presql $::G(perm:presql)} set presql } + +proc wal_is_ok {} { + if { [forced_proxy_locking] } { + return 1 + } + if { ![path_is_local "."] } { + return 0 + } + if { [path_is_dos "."] } { + return 0 + } + return 1 +} proc isquick {} { set ret 0 catch {set ret $::G(isquick)} set ret Index: test/tkt-2d1a5c67d.test ================================================================== --- test/tkt-2d1a5c67d.test +++ test/tkt-2d1a5c67d.test @@ -19,10 +19,11 @@ source $testdir/tester.tcl set testprefix tkt-2d1a5c67d ifcapable {!vtab} {finish_test; return} if {[wal_is_capable]==0} {finish_test; return} +if {![wal_is_ok]} {finish_test; return} for {set ii 1} {$ii<=10} {incr ii} { do_test tkt-2d1a5c67d.1.$ii { db close forcedelete test.db test.db-wal Index: test/tkt-313723c356.test ================================================================== --- test/tkt-313723c356.test +++ test/tkt-313723c356.test @@ -17,10 +17,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl if {![wal_is_capable]} { finish_test ; return } +if ![wal_is_ok] { finish_test; return } do_execsql_test tkt-313723c356.1 { PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); Index: test/tkt3457.test ================================================================== --- test/tkt3457.test +++ test/tkt3457.test @@ -72,31 +72,32 @@ catch { test_syscall install fchmod test_syscall fault 1 1 } -do_test tkt3457-1.2 { - forcecopy bak.db-journal test.db-journal - file attributes test.db-journal -permissions --------- - catchsql { SELECT * FROM t1 } -} {1 {unable to open database file}} -do_test tkt3457-1.3 { - forcecopy bak.db-journal test.db-journal - file attributes test.db-journal -permissions -w--w--w- - catchsql { SELECT * FROM t1 } -} {1 {unable to open database file}} -do_test tkt3457-1.4 { - forcecopy bak.db-journal test.db-journal - file attributes test.db-journal -permissions r--r--r-- - catchsql { SELECT * FROM t1 } -} {1 {unable to open database file}} - -do_test tkt3457-1.5 { - forcecopy bak.db-journal test.db-journal - file attributes test.db-journal -permissions rw-rw-rw- - catchsql { SELECT * FROM t1 } -} {0 {1 2 3 4 5 6}} +if { ![path_is_dos "."] } { + do_test tkt3457-1.2 { + forcecopy bak.db-journal test.db-journal + file attributes test.db-journal -permissions --------- + catchsql { SELECT * FROM t1 } + } {1 {unable to open database file}} + do_test tkt3457-1.3 { + forcecopy bak.db-journal test.db-journal + file attributes test.db-journal -permissions -w--w--w- + catchsql { SELECT * FROM t1 } + } {1 {unable to open database file}} + do_test tkt3457-1.4 { + forcecopy bak.db-journal test.db-journal + file attributes test.db-journal -permissions r--r--r-- + catchsql { SELECT * FROM t1 } + } {1 {unable to open database file}} + do_test tkt3457-1.5 { + forcecopy bak.db-journal test.db-journal + file attributes test.db-journal -permissions rw-rw-rw- + catchsql { SELECT * FROM t1 } + } {0 {1 2 3 4 5 6}} +} # Reenable fchmod catch { test_syscall uninstall test_syscall fault 0 0 Index: test/wal.test ================================================================== --- test/wal.test +++ test/wal.test @@ -20,10 +20,14 @@ source $testdir/wal_common.tcl set testprefix wal ifcapable !wal {finish_test ; return } +if { ![wal_is_ok] } { + finish_test + return +} test_set_config_pagecache 0 0 proc reopen_db {} { catch { db close } forcedelete test.db test.db-wal test.db-wal-summary @@ -171,10 +175,13 @@ SELECT * FROM t1; } } {a b} do_test wal-4.4.1 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close sqlite3 db test.db db func blob blob list [execsql { SELECT * FROM t1 }] [file size test.db-wal] } {{a b} 0} @@ -222,10 +229,13 @@ } {ok} db2 close do_test wal-4.5.1 { reopen_db + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db func blob blob execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES('a', 'b'); @@ -336,10 +346,13 @@ sqlite3_simulate_device -sectorsize $sector foreach pgsz {512 1024 2048 4096} { forcedelete test.db test.db-wal do_test wal-6.$sector.$pgsz.1 { sqlite3 db test.db -vfs devsym + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql " PRAGMA page_size = $pgsz; PRAGMA auto_vacuum = 0; PRAGMA journal_mode = wal; " @@ -722,10 +735,13 @@ do_test wal-11.8 { execsql { PRAGMA wal_checkpoint } list [expr [file size test.db]/1024] [file size test.db-wal] } [list 37 [wal_file_size 40 1024]] do_test wal-11.9 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db @@ -736,10 +752,13 @@ #set nWal 39 #if {[permutation]!="mmap"} {set nWal 37} #ifcapable !mmap {set nWal 37} set nWal 34 +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; BEGIN; INSERT INTO t1 SELECT blob(900) FROM t1; -- 32 @@ -783,10 +802,13 @@ INSERT INTO t1 VALUES('A', 1); } list [expr [file size test.db]/1024] [file size test.db-wal] } [list 1 [wal_file_size 5 1024]] do_test wal-12.2 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close sqlite3 db test.db execsql { PRAGMA synchronous = normal; UPDATE t1 SET y = 0 WHERE x = 'A'; @@ -820,10 +842,13 @@ forcecopy test.db-wal test2.db-wal sqlite3_wal db2 test2.db execsql { SELECT * FROM t2 } db2 } {B 2} db2 close +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} db close #------------------------------------------------------------------------- # Check a fun corruption case has been fixed. # @@ -977,14 +1002,16 @@ ATTACH 'test2.db' AS aux; PRAGMA main.auto_vacuum = 0; PRAGMA aux.auto_vacuum = 0; PRAGMA main.journal_mode = WAL; PRAGMA aux.journal_mode = WAL; + SELECT count(*) FROM main.sqlite_master, aux.sqlite_master; PRAGMA main.synchronous = NORMAL; PRAGMA aux.synchronous = NORMAL; + PRAGMA aux.synchronous = FULL; } - } {wal wal} + } {wal wal 0} do_test wal-16.$tn.2 { execsql { CREATE TABLE main.t1(a, b, PRIMARY KEY(a, b)); CREATE TABLE aux.t2(a, b, PRIMARY KEY(a, b)); @@ -1051,10 +1078,11 @@ execsql { PRAGMA auto_vacuum = 0; PRAGMA page_size = 512; PRAGMA cache_size = -2000; PRAGMA journal_mode = WAL; + SELECT * FROM sqlite_master; PRAGMA synchronous = FULL; } execsql { BEGIN; CREATE TABLE t(x); @@ -1266,10 +1294,13 @@ INSERT INTO t1 VALUES(5, 6); SELECT * FROM t1; } } {1 2 3 4 5 6} do_test wal-19.3 { + ifcapable enable_persist_wal { + file_control_persist_wal db2 0 + } db close db2 close file exists test.db-wal } {0} do_test wal-19.4 { @@ -1420,10 +1451,13 @@ CREATE TABLE t1(a, b); PRAGMA journal_mode = WAL; INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); } + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } faultsim_save_and_close sqlite3_shutdown test_sqlite3_log [list lappend ::log] set ::log [list] @@ -1466,10 +1500,13 @@ INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; INSERT INTO t1 SELECT * FROM t1; } {wal} do_test 24.2 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql { DELETE FROM t1; PRAGMA wal_checkpoint; } db close @@ -1494,10 +1531,13 @@ do_test 24.5 { file size test.db-wal } [wal_file_size 1 1024] } +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} db close sqlite3_shutdown test_sqlite3_log sqlite3_initialize Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -20,10 +20,11 @@ source $testdir/wal_common.tcl set testprefix wal2 ifcapable !wal {finish_test ; return } +if ![wal_is_ok] {finish_test ; return } set sqlite_sync_count 0 proc cond_incr_sync_count {adj} { global sqlite_sync_count if {$::tcl_platform(platform) == "windows"} { @@ -68,10 +69,17 @@ set v [lindex $ints $idx] incr v $incrval lset ints $idx $v set_tvfs_hdr $file $ints } + +set shmpath test.db-shm +if {[forced_proxy_locking]} { + sqlite3 db test.db + set shmpath [execsql { pragma lock_proxy_file }]-shm + db close +} #------------------------------------------------------------------------- # Test case wal2-1.*: # @@ -571,13 +579,19 @@ COMMIT; } list [file exists test.db-wal] [file exists test.db-journal] } {1 0} do_test wal2-6.3.2 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql { PRAGMA journal_mode = DELETE } file exists test.db-wal } {0} +do_test wal2-6.3.2.1 { + execsql { PRAGMA journal_mode; } +} {delete} do_test wal2-6.3.3 { execsql { PRAGMA lock_status } } {main exclusive temp closed} do_test wal2-6.3.4 { execsql { @@ -728,11 +742,13 @@ set ::locks [list] do_test wal2-6.4.$tn.1 { execsql $S } $res do_test wal2-6.4.$tn.2 { set ::locks } $L } - +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} db close tvfs delete do_test wal2-6.5.1 { sqlite3 db test.db @@ -959,10 +975,13 @@ CREATE TABLE t1(a, b); PRAGMA wal_checkpoint; INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); } + if {[forced_proxy_locking]} { + forcecopy $shmpath sv_test.db-shm + } faultsim_save_and_close } {} do_test wal2-10.1.2 { faultsim_restore_and_reopen execsql { SELECT * FROM t1 } @@ -1058,10 +1077,15 @@ # If a connection is required to create a WAL or SHM file, it creates # the new files with the same file-system permissions as the database # file itself. Test this. # if {$::tcl_platform(platform) == "unix"} { + if {[forced_proxy_locking]} { + # faultsim_delete_and_reopen doesn't know about the shm file redirect... + forcedelete $shmpath + } + faultsim_delete_and_reopen # Changed on 2012-02-13: umask is deliberately ignored for -wal files. #set umask [exec /bin/sh -c umask] set umask 0 @@ -1071,11 +1095,11 @@ execsql { CREATE TABLE tx(y, z); PRAGMA journal_mode = WAL; } db close - list [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db-wal] [file exists $shmpath] } {0 0} foreach {tn permissions} { 1 00644 2 00666 @@ -1086,24 +1110,27 @@ do_test wal2-12.2.$tn.1 { file attributes test.db -permissions $permissions string map {o 0} [file attributes test.db -permissions] } $permissions do_test wal2-12.2.$tn.2 { - list [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db-wal] [file exists $shmpath] } {0 0} do_test wal2-12.2.$tn.3 { sqlite3 db test.db execsql { INSERT INTO tx DEFAULT VALUES } - list [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db-wal] [file exists $shmpath] } {1 1} do_test wal2-12.2.$tn.4 { - set x [list [file attr test.db-wal -perm] [file attr test.db-shm -perm]] + set x [list [file attr test.db-wal -perm] [file attr $shmpath -perm]] string map {o 0} $x } [list $effective $effective] do_test wal2-12.2.$tn.5 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close - list [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db-wal] [file exists $shmpath] } {0 0} } } #------------------------------------------------------------------------- @@ -1112,30 +1139,37 @@ # read-only. # if {$::tcl_platform(platform) == "unix"} { proc perm {} { set L [list] - foreach f {test.db test.db-wal test.db-shm} { + foreach f {test.db test.db-wal $shmpath} { if {[file exists $f]} { lappend L [file attr $f -perm] } else { lappend L {} } } set L } + if {[forced_proxy_locking]} { + # faultsim_delete_and_reopen doesn't know about the shm file redirect... + forcedelete $shmpath + } faultsim_delete_and_reopen execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); PRAGMA wal_checkpoint; INSERT INTO t1 VALUES('3.14', '2.72'); } do_test wal2-13.1.1 { - list [file exists test.db-shm] [file exists test.db-wal] + list [file exists $shmpath] [file exists test.db-wal] } {1 1} + if {[forced_proxy_locking]} { + forcecopy $shmpath proxysv_test.db-shm + } faultsim_save_and_close foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} { 2 00644 00644 00644 1 1 1 3 00644 00400 00644 1 1 0 @@ -1145,18 +1179,21 @@ 7 00644 00000 00644 1 0 0 8 00644 00644 00000 1 0 0 9 00000 00644 00644 0 0 0 } { faultsim_restore + if {[forced_proxy_locking]} { + forcecopy proxysv_test.db-shm $shmpath + } do_test wal2-13.$tn.1 { file attr test.db -perm $db_perm file attr test.db-wal -perm $wal_perm - file attr test.db-shm -perm $shm_perm + file attr $shmpath -perm $shm_perm set L [file attr test.db -perm] lappend L [file attr test.db-wal -perm] - lappend L [file attr test.db-shm -perm] + lappend L [file attr $shmpath -perm] string map {o 0} $L } [list $db_perm $wal_perm $shm_perm] # If $can_open is true, then it should be possible to open a database # handle. Otherwise, if $can_open is 0, attempting to open the db @@ -1167,10 +1204,14 @@ do_test wal2-13.$tn.2 { list [catch {sqlite3 db test.db ; set {} ok} msg] $msg } $r($can_open) if {$can_open} { + # Different behavior, because Darwin does an access() call prior + # to attempting to open the SHM in read/write mode and demotes to + # read-only if the SHM is read-only. + if {$tn==4 && $::tcl_platform(os)=="Darwin"} {set can_read 1} # If $can_read is true, then the client should be able to read from # the database file. If $can_read is false, attempting to read should # throw the "unable to open database file" exception. # @@ -1216,10 +1257,12 @@ do_execsql_test wal2-14.$tn.0 { PRAGMA page_size = 4096 } {} do_execsql_test wal2-14.$tn.1 { PRAGMA journal_mode = WAL } {wal} set sqlite_sync_count 0 set sqlite_fullsync_count 0 + + set useres $reslist do_execsql_test wal2-14.$tn.2 { PRAGMA wal_autocheckpoint = 10; CREATE TABLE t1(a, b); -- 2 wal syncs INSERT INTO t1 VALUES(1, 2); -- 2 wal sync @@ -1232,19 +1275,19 @@ } {10 0 3 3 0 1 1} do_test wal2-14.$tn.3 { cond_incr_sync_count 1 list $sqlite_sync_count $sqlite_fullsync_count - } [lrange $reslist 0 1] + } [lrange $useres 0 1] set sqlite_sync_count 0 set sqlite_fullsync_count 0 do_test wal2-14.$tn.4 { execsql { INSERT INTO t1 VALUES(7, zeroblob(12*4096)) } list $sqlite_sync_count $sqlite_fullsync_count - } [lrange $reslist 2 3] + } [lrange $useres 2 3] set sqlite_sync_count 0 set sqlite_fullsync_count 0 do_test wal2-14.$tn.5 { @@ -1252,11 +1295,11 @@ execsql { INSERT INTO t1 VALUES(9, 10) } execsql { INSERT INTO t1 VALUES(11, 12) } execsql { INSERT INTO t1 VALUES(13, 14) } db close list $sqlite_sync_count $sqlite_fullsync_count - } [lrange $reslist 4 5] + } [lrange $useres 4 5] } catch { db close } # PRAGMA checkpoint_fullsync Index: test/wal3.test ================================================================== --- test/wal3.test +++ test/wal3.test @@ -17,10 +17,11 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl source $testdir/malloc_common.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] {finish_test ; return } set a_string_counter 1 proc a_string {n} { global a_string_counter incr a_string_counter @@ -209,10 +210,12 @@ proc sync_counter {args} { foreach {method filename id flags} $args break lappend ::syncs [file tail $filename] $flags } + set usecount $synccount + do_test wal3-3.$tn { forcedelete test.db test.db-wal test.db-journal testvfs T T filter {} @@ -231,11 +234,11 @@ INSERT INTO x VALUES('z'); PRAGMA wal_checkpoint; } T filter {} set ::syncs - } $synccount + } $usecount db close T delete } Index: test/wal4.test ================================================================== --- test/wal4.test +++ test/wal4.test @@ -14,10 +14,14 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl ifcapable !wal {finish_test ; return } +if { ![wal_is_ok] || [path_is_dos "."]} { + finish_test + return +} do_test wal4-1.1 { execsql { PRAGMA journal_mode=WAL; CREATE TABLE t1(x); Index: test/wal5.test ================================================================== --- test/wal5.test +++ test/wal5.test @@ -16,10 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } do_not_use_codec set testprefix wal5 proc db_page_count {{file test.db}} { expr [file size $file] / 1024 } @@ -78,10 +79,15 @@ eval $::busy_handler_script return 0 } proc reopen_all {} { + ifcapable enable_persist_wal { + code1 { file_control_persist_wal db 0 } + code2 { file_control_persist_wal db2 0 } + code3 { file_control_persist_wal db3 0 } + } code1 {db close} code2 {db2 close} code3 {db3 close} code1 {sqlite3 db test.db} @@ -341,10 +347,15 @@ do_test 3.$tn.3 { code2 { do_wal_checkpoint db2 } } {0 2 2} do_test 3.$tn.4 { code3 { do_wal_checkpoint db3 } } {0 2 2} + ifcapable enable_persist_wal { + code1 { file_control_persist_wal db 0 } + code2 { file_control_persist_wal db2 0 } + code3 { file_control_persist_wal db3 0 } + } code1 {db close} code2 {db2 close} code3 {db3 close} code1 {sqlite3 db test.db} Index: test/wal6.test ================================================================== --- test/wal6.test +++ test/wal6.test @@ -18,10 +18,12 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl source $testdir/malloc_common.tcl ifcapable !wal {finish_test ; return } + +if { ![wal_is_ok] } {finish_test ; return } #------------------------------------------------------------------------- # Changing to WAL mode in one connection forces the change in others. # db close Index: test/wal7.test ================================================================== --- test/wal7.test +++ test/wal7.test @@ -14,10 +14,11 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } # Case 1: No size limit. Journal can get large. # do_test wal7-1.0 { db close Index: test/wal8.test ================================================================== --- test/wal8.test +++ test/wal8.test @@ -24,11 +24,13 @@ # is a no-op. # set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix wal8 + ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } do_not_use_codec db close forcedelete test.db test.db-wal Index: test/wal_common.tcl ================================================================== --- test/wal_common.tcl +++ test/wal_common.tcl @@ -30,10 +30,24 @@ foreach {v1 v2} $intlist { set c1 [expr {($c1 + $v1 + $c2)&0xFFFFFFFF}] set c2 [expr {($c2 + $v2 + $c1)&0xFFFFFFFF}] } } + +# If the synchronous mode for the main database of db handle $db +# is either OFF or NORMAL, return $nRight. Otherwise, if it is +# FULL, return $nWrite+$nTrans. +# +proc wal_frames {db nWrite nTrans} { + set nRet $nWrite + switch -- [$db one {PRAGMA main.synchronous}] { + 0 { } + 1 { } + default { incr nRet $nTrans } + } + set nRet +} # This proc calculates checksums in the same way as those used by SQLite # in WAL files. If the $endian argument is "big", then checksums are # calculated by interpreting data as an array of big-endian integers. If Index: test/walbak.test ================================================================== --- test/walbak.test +++ test/walbak.test @@ -19,10 +19,11 @@ source $testdir/malloc_common.tcl do_not_use_codec ifcapable !wal {finish_test ; return } +if { ![wal_is_ok] } { finish_test ; return } # Test organization: # # walback-1.*: Simple tests. @@ -322,18 +323,24 @@ forcedelete test.db test.db2 do_test walbak-4.$tn.1 { sqlite3 db test.db db eval "PRAGMA journal_mode = $src" + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db eval { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES('I', 'II'); INSERT INTO t1 VALUES('III', 'IV'); } sqlite3 db2 test.db2 db2 eval "PRAGMA journal_mode = $dest" + ifcapable enable_persist_wal { + file_control_persist_wal db2 0 + } db2 eval { CREATE TABLE t2(x, y); INSERT INTO t2 VALUES('1', '2'); INSERT INTO t2 VALUES('3', '4'); } Index: test/walbig.test ================================================================== --- test/walbig.test +++ test/walbig.test @@ -19,10 +19,11 @@ ifcapable !wal { finish_test return } +if ![wal_is_ok] { finish_test; return } # Do not use a codec for this file, as the database is manipulated using # external methods (the [fake_big_file] and [hexio_write] commands). # do_not_use_codec @@ -29,10 +30,14 @@ # If SQLITE_DISABLE_LFS is defined, omit this file. ifcapable !lfs { finish_test return +} +if { ![wal_is_ok] } { + finish_test + return } set a_string_counter 1 proc a_string {n} { incr ::a_string_counter Index: test/walcksum.test ================================================================== --- test/walcksum.test +++ test/walcksum.test @@ -14,10 +14,11 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } # Read and return the contents of file $filename. Treat the content as # binary data. # proc readfile {filename} { Index: test/walcrash.test ================================================================== --- test/walcrash.test +++ test/walcrash.test @@ -26,10 +26,11 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } db close set seed 0 set REPEATS 100 Index: test/walcrash2.test ================================================================== --- test/walcrash2.test +++ test/walcrash2.test @@ -14,10 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } +if { ![wal_is_ok] } { finish_test ; return } #------------------------------------------------------------------------- # This test case demonstrates a flaw in the wal-index manipulation that # existed at one point: If a process crashes mid-transaction, it may have Index: test/walcrash3.test ================================================================== --- test/walcrash3.test +++ test/walcrash3.test @@ -16,10 +16,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } + set testprefix walcrash3 db close testvfs tvfs tvfs filter {xTruncate xWrite} Index: test/walfault.test ================================================================== --- test/walfault.test +++ test/walfault.test @@ -17,10 +17,11 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl source $testdir/lock_common.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } #------------------------------------------------------------------------- # This test case, walfault-1-*, simulates faults while executing a # # PRAGMA journal_mode = WAL; Index: test/walhook.test ================================================================== --- test/walhook.test +++ test/walhook.test @@ -20,10 +20,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/wal_common.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } set ::wal_hook [list] proc wal_hook {zDb nEntry} { lappend ::wal_hook $zDb $nEntry return 0 @@ -69,12 +70,18 @@ execsql { CREATE TABLE t3(a PRIMARY KEY, b) } file size test.db } [expr 6*1024] db2 close +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} db close sqlite3 db test.db +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} do_test walhook-2.1 { execsql { PRAGMA synchronous = NORMAL } execsql { PRAGMA wal_autocheckpoint } } {1000} do_test walhook-2.2 { Index: test/walmode.test ================================================================== --- test/walmode.test +++ test/walmode.test @@ -33,10 +33,24 @@ } {delete} finish_test return } +if { ![wal_is_ok] && ![path_is_dos "."]} { + do_test walmode-0.1 { + execsql { PRAGMA journal_mode = wal } + } {delete} + do_test walmode-0.2 { + execsql { PRAGMA main.journal_mode = wal } + } {delete} + do_test walmode-0.3 { + execsql { PRAGMA main.journal_mode } + } {delete} + + finish_test + return +} do_test walmode-1.1 { set sqlite_sync_count 0 execsql { PRAGMA page_size = 1024 } execsql { PRAGMA journal_mode = wal } @@ -43,16 +57,22 @@ } {wal} do_test walmode-1.2 { file size test.db } {1024} -if {[atomic_batch_write test.db]==0} { - set expected_sync_count 3 - if {$::tcl_platform(platform)!="windows"} { - ifcapable dirsync { - incr expected_sync_count - } +# Determine how many sync() calls to expect from the "journal_mode=WAL" +# command above. Note that if DEFAULT_WAL_SAFETYLEVEL is defined, the +# safety-level may have been modified while compiling the "journal_mode=WAL" +# statement. +switch -- [db eval {PRAGMA main.synchronous}] { + 0 { set expected_sync_count 0 } + 1 { set expected_sync_count 2 } + default { set expected_sync_count 3 } +} +if {$::tcl_platform(platform)!="windows"} { + ifcapable dirsync { + incr expected_sync_count } do_test walmode-1.3 { set sqlite_sync_count } $expected_sync_count } @@ -66,10 +86,13 @@ } {1024} do_test walmode-1.6 { file exists test.db-wal } {1} do_test walmode-1.7 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close file exists test.db-wal } {0} # There is now a database file with the read and write versions set to 2 @@ -82,10 +105,13 @@ do_test walmode-2.2 { execsql { SELECT * FROM sqlite_master } file exists test.db-wal } {1} do_test walmode-2.3 { + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } db close file exists test.db-wal } {0} # If the first statement executed is "PRAGMA journal_mode = wal", and @@ -104,10 +130,13 @@ # Test that changing back to journal_mode=persist works. # do_test walmode-4.1 { execsql { INSERT INTO t1 VALUES(1, 2) } + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql { PRAGMA journal_mode = persist } } {persist} if {[atomic_batch_write test.db]==0} { do_test walmode-4.2 { list [file exists test.db-journal] [file exists test.db-wal] @@ -131,10 +160,13 @@ # from WAL to rollback mode because a second connection has the database # open. Or from rollback to WAL. # do_test walmode-4.6 { sqlite3 db2 test.db + ifcapable enable_persist_wal { + file_control_persist_wal db2 0 + } execsql { PRAGMA main.journal_mode } db2 } {delete} do_test walmode-4.7 { execsql { PRAGMA main.journal_mode = wal } db } {wal} @@ -148,10 +180,13 @@ execsql { PRAGMA main.journal_mode } db } {wal} do_test walmode-4.11 { db2 close + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql { PRAGMA journal_mode = delete } db } {delete} do_test walmode-4.12 { execsql { PRAGMA main.journal_mode } db } {delete} @@ -301,10 +336,13 @@ # loaded when sqlite3_prepare_v2() is called to compile the statement. # do_test walmode-7.0 { forcedelete test.db sqlite3 db test.db + ifcapable enable_persist_wal { + file_control_persist_wal db 0 + } execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); } } {wal} Index: test/walnoshm.test ================================================================== --- test/walnoshm.test +++ test/walnoshm.test @@ -15,10 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix walnoshm ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } db close testvfs tvfsshm testvfs tvfs -default 1 -iversion 1 sqlite3 db test.db @@ -51,10 +52,13 @@ do_execsql_test 1.6 { INSERT INTO t1 VALUES(3, 4) } do_execsql_test 1.7 { PRAGMA locking_mode = normal; } {exclusive} +ifcapable enable_persist_wal { + file_control_persist_wal db 0 +} do_execsql_test 1.8 { PRAGMA journal_mode = delete; PRAGMA main.locking_mode; } {delete exclusive} do_execsql_test 1.9 { Index: test/walpersist.test ================================================================== --- test/walpersist.test +++ test/walpersist.test @@ -19,10 +19,16 @@ ifcapable !wal { finish_test return } +if ![wal_is_ok] { finish_test; return } + +set shmpath test.db-shm +if {[forced_proxy_locking]} { + set shmpath [execsql { pragma lock_proxy_file }]-shm +} do_test walpersist-1.0 { db eval { PRAGMA journal_mode=WAL; CREATE TABLE t1(a); @@ -29,26 +35,39 @@ INSERT INTO t1 VALUES(randomblob(5000)); } file exists test.db-wal } {1} do_test walpersist-1.1 { - file exists test.db-shm + file exists $shmpath } {1} -do_test walpersist-1.2 { - db close - list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] -} {1 0 0} +ifcapable enable_persist_wal { + do_test walpersist-1.2-on { + db close + list [file exists test.db] [file exists test.db-wal] [file exists $shmpath] + } {1 1 1} +} { + do_test walpersist-1.2-off { + db close + list [file exists test.db] [file exists test.db-wal] [file exists $shmpath] + } {1 0 0} +} do_test walpersist-1.3 { sqlite3 db test.db db eval {SELECT length(a) FROM t1} } {5000} do_test walpersist-1.4 { - list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db] [file exists test.db-wal] [file exists $shmpath] } {1 1 1} -do_test walpersist-1.5 { - file_control_persist_wal db -1 -} {0 0} +ifcapable enable_persist_wal { + do_test walpersist-1.5-on { + file_control_persist_wal db -1 + } {0 1} +} { + do_test walpersist-1.5-off { + file_control_persist_wal db -1 + } {0 0} +} do_test walpersist-1.6 { file_control_persist_wal db 1 } {0 1} do_test walpersist-1.7 { file_control_persist_wal db -1 @@ -62,19 +81,19 @@ do_test walpersist-1.10 { file_control_persist_wal db 1 } {0 1} do_test walpersist-1.11 { db close - list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db] [file exists test.db-wal] [file exists $shmpath] } {1 1 1} # Make sure the journal_size_limit works to limit the size of the # persisted wal file. In persistent-wal mode, any non-negative # journal_size_limit causes the WAL file to be truncated to zero bytes # when closing. # -forcedelete test.db test.db-shm test.db-wal +forcedelete test.db $shmpath test.db-wal do_test walpersist-2.1 { sqlite3 db test.db db eval { PRAGMA journal_mode=WAL; PRAGMA wal_autocheckpoint=OFF; @@ -95,11 +114,11 @@ execsql { PRAGMA integrity_check } } {ok} do_test 3.1 { catch {db close} - forcedelete test.db test.db-shm test.db-wal + forcedelete test.db $shmpath test.db-wal sqlite3 db test.db execsql { PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint=128; Index: test/walro.test ================================================================== --- test/walro.test +++ test/walro.test @@ -28,10 +28,18 @@ # ifcapable !wal { finish_test return } +if ![wal_is_ok] { finish_test; return } + +set shmpath test.db-shm +if {[forced_proxy_locking]} { + sqlite3 db test.db + set shmpath [execsql { pragma lock_proxy_file }]-shm + db close +} do_multiclient_test tn { # Close all connections and delete the database. # @@ -60,15 +68,15 @@ PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; CREATE TABLE t1(x, y); INSERT INTO t1 VALUES('a', 'b'); } - file exists test.db-shm + file exists $shmpath } {1} do_test 1.1.2 { - file attributes test.db-shm -permissions r--r--r-- + file attributes $shmpath -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } } {} do_test 1.1.3 { sql1 "SELECT * FROM t1" } {a b} do_test 1.1.4 { sql2 "INSERT INTO t1 VALUES('c', 'd')" } {} @@ -97,36 +105,36 @@ do_test 1.1.13 { sql2 "INSERT INTO t1 VALUES('i', 'j')" } {} do_test 1.2.1 { code2 { db2 close } code1 { db close } - list [file exists test.db-wal] [file exists test.db-shm] + list [file exists test.db-wal] [file exists $shmpath] } {1 1} do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } list [catch { sql1 { SELECT * FROM t1 } } msg] $msg } {0 {a b c d e f g h i j}} do_test 1.2.3 { code1 { db close } - file attributes test.db-shm -permissions rw-r--r-- - hexio_write test.db-shm 0 01020304 - file attributes test.db-shm -permissions r--r--r-- + file attributes $shmpath -permissions rw-r--r-- + hexio_write $shmpath 0 01020304 + file attributes $shmpath -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } } {0 {a b c d e f g h i j}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } } {SQLITE_OK} do_test 1.2.5 { - file attributes test.db-shm -permissions rw-r--r-- + file attributes $shmpath -permissions rw-r--r-- code2 { sqlite3 db2 test.db } sql2 "SELECT * FROM t1" } {a b c d e f g h i j} - file attributes test.db-shm -permissions r--r--r-- + file attributes $shmpath -permissions r--r--r-- do_test 1.2.6 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j} do_test 1.2.7 { sql2 { PRAGMA wal_checkpoint; @@ -150,22 +158,26 @@ } {0 {a b c d e f g h i j k l}} # Also test that if the -shm file can be opened for read/write access, # it is not if readonly_shm=1 is present in the URI. do_test 1.3.2.1 { + ifcapable enable_persist_wal { + code1 { file_control_persist_wal db 0 } + code2 { file_control_persist_wal db2 0 } + } code1 { db close } code2 { db2 close } - file exists test.db-shm + file exists $shmpath } {0} do_test 1.3.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM sqlite_master } } {1 {unable to open database file}} do_test 1.3.2.3 { code1 { db close } - close [open test.db-shm w] - file attributes test.db-shm -permissions r--r--r-- + close [open $shmpath w] + file attributes $shmpath -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } } {0 {a b c d e f g h i j k l}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } @@ -295,6 +307,71 @@ code1 { db close } code1 { tv delete } } {} } +forcedelete $shmpath + +if {$tcl_platform(os)=="Darwin"} { + ifcapable enable_persist_wal { + + #-------------------------------------------------------------------------- + + catch {db2 close} + reset_db + do_execsql_test 3.1 { + CREATE TABLE t1(a, b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1, 2); + } {wal} + db_save + db close + db_restore + + sqlite3 db test.db -readonly 1 + + do_execsql_test 3.2 { SELECT * FROM t1 } {1 2} + do_catchsql_test 3.3 { + INSERT INTO t1 VALUES(3, 4) + } {1 {attempt to write a readonly database}} + + sqlite3 db2 test.db + do_test 3.4 { + db2 eval { INSERT INTO t1 VALUES(3, 4) } + } {} + do_execsql_test 3.5 { SELECT * FROM t1 } {1 2 3 4} + + db close + db2 close + db_restore + file attributes $shmpath -permissions r--r--r-- + sqlite3 db test.db -readonly 1 + do_execsql_test 3.6 { SELECT * FROM t1 } {1 2} + + db close + db_restore + file attributes $shmpath -permissions r--r--r-- + sqlite3 db test.db + do_test 3.7 { + catchsql { SELECT * FROM t1 } + } {1 {unable to open database file}} + + db close + db_restore + file attributes $shmpath -permissions r--r--r-- + sqlite3 db test.db -readonly 1 + do_execsql_test 3.8 { SELECT * FROM t1 } {1 2} + + sqlite3 db2 test.db + do_test 3.9 { db2 eval { SELECT * FROM t1 } } {1 2} + do_test 3.10 { + catchsql { INSERT INTO t1 VALUES(3, 4) } db2 + } {1 {attempt to write a readonly database}} + + catch { db close } + catch { db2 close } + + forcedelete $shmpath + } ;# endif capable enable_persist_wal +} ;# endif os Darwin + finish_test Index: test/walshared.test ================================================================== --- test/walshared.test +++ test/walshared.test @@ -15,10 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl ifcapable !wal {finish_test ; return } +if ![wal_is_ok] { finish_test; return } db close set ::enable_shared_cache [sqlite3_enable_shared_cache 1] sqlite3 db test.db Index: test/walslow.test ================================================================== --- test/walslow.test +++ test/walslow.test @@ -18,11 +18,11 @@ source $testdir/tester.tcl source $testdir/wal_common.tcl source $testdir/lock_common.tcl ifcapable !wal {finish_test ; return } - +if ![wal_is_ok] { finish_test; return } set testprefix walslow proc reopen_db {} { catch { db close } forcedelete test.db test.db-wal Index: test/walthread.test ================================================================== --- test/walthread.test +++ test/walthread.test @@ -17,10 +17,11 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl if {[run_thread_tests]==0} { finish_test ; return } ifcapable !wal { finish_test ; return } +if ![wal_is_ok] { finish_test; return } set sqlite_walsummary_mmap_incr 64 # How long, in seconds, to run each test for. If a test is set to run for # 0 seconds, it is omitted entirely. Index: test/window1.test ================================================================== --- test/window1.test +++ test/window1.test @@ -1590,11 +1590,10 @@ do_execsql_test 48.1 { SELECT (SELECT max(x)OVER(ORDER BY x) + min(x)OVER(ORDER BY x)) FROM (SELECT (SELECT sum(a) FROM t1 GROUP BY a) AS x FROM t1); } {2 2 2} - #------------------------------------------------------------------------- reset_db do_execsql_test 49.1 { CREATE TABLE t1 (a PRIMARY KEY); INSERT INTO t1 VALUES(1); Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -120,10 +120,11 @@ sqlite3ext.h sqlite3rbu.h sqliteicu.h sqliteInt.h sqliteLimit.h + sqlrr.h vdbe.h vdbeInt.h vxworks.h wal.h whereInt.h @@ -395,10 +396,11 @@ json1.c rtree.c icu.c fts3_icu.c + sqlrr.c sqlite3rbu.c dbstat.c dbpage.c sqlite3session.c fts5.c