Index: src/sqlite3.c ================================================================== --- src/sqlite3.c +++ src/sqlite3.c @@ -1,12 +1,12 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.7.0. By combining all the individual C code files into this +** version 3.7.4. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a one translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements -** of 5% are more are commonly seen when SQLite is compiled as a single +** of 5% or more are commonly seen when SQLite is compiled as a single ** translation unit. ** ** This file is all you need to compile SQLite. To use SQLite in other ** programs, you need this file and the "sqlite3.h" header file that defines ** the programming interface to the SQLite library. (If you do not have @@ -213,24 +213,25 @@ */ #ifndef SQLITE_MAX_VARIABLE_NUMBER # define SQLITE_MAX_VARIABLE_NUMBER 999 #endif -/* Maximum page size. The upper bound on this value is 32768. This a limit -** imposed by the necessity of storing the value in a 2-byte unsigned integer -** and the fact that the page size must be a power of 2. +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. ** -** If this limit is changed, then the compiled library is technically -** incompatible with an SQLite library compiled with a different limit. If -** a process operating on a database with a page-size of 65536 bytes -** crashes, then an instance of SQLite compiled with the default page-size -** limit will not be able to rollback the aborted transaction. This could -** lead to database corruption. +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. */ -#ifndef SQLITE_MAX_PAGE_SIZE -# define SQLITE_MAX_PAGE_SIZE 32768 +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE #endif +#define SQLITE_MAX_PAGE_SIZE 65536 /* ** The default size of a database page. */ @@ -351,19 +352,25 @@ # define SQLITE_INT_TO_PTR(X) ((void*)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif /* -** The SQLITE_THREADSAFE macro must be defined as either 0 or 1. +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the libary is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** ** Older versions of SQLite used an optional THREADSAFE macro. -** We support that for legacy +** We support that for legacy. */ #if !defined(SQLITE_THREADSAFE) #if defined(THREADSAFE) # define SQLITE_THREADSAFE THREADSAFE #else -# define SQLITE_THREADSAFE 1 +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ #endif #endif /* ** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. @@ -514,11 +521,11 @@ /* ** Return true (non-zero) if the input is a integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. */ -#define IS_BIG_INT(X) (((X)&(i64)0xffffffff)!=0) +#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) /* ** The macro unlikely() is a hint that surrounds a boolean ** expression that is usually false. Macro likely() surrounds ** a boolean expression that is usually true. GCC is able to @@ -631,23 +638,23 @@ ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** ** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management -** system. ^The SQLITE_SOURCE_ID macro evalutes to +** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and an SHA1 ** hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.0" -#define SQLITE_VERSION_NUMBER 3007000 -#define SQLITE_SOURCE_ID "2010-07-09 12:57:54 0c32c4bbdd74297767dcf4ec4295f9cc72875af0" +#define SQLITE_VERSION "3.7.4" +#define SQLITE_VERSION_NUMBER 3007004 +#define SQLITE_SOURCE_ID "2010-12-06 21:09:59 fabcb6b95e1d4059d1e6c6183f65846f6cbd5749" /* ** CAPI3REF: Run-Time Library Version Numbers ** KEYWORDS: sqlite3_version, sqlite3_sourceid ** @@ -688,19 +695,19 @@ ** ^The sqlite3_compileoption_used() function returns 0 or 1 ** indicating whether the specified option was defined at ** compile time. ^The SQLITE_ prefix may be omitted from the ** option name passed to sqlite3_compileoption_used(). ** -** ^The sqlite3_compileoption_get() function allows interating +** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, ** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ ** prefix is omitted from any strings returned by ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifing the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and ** [sqlite_compileoption_get()] and the [compile_options pragma]. */ @@ -802,11 +809,11 @@ /* ** CAPI3REF: Closing A Database Connection ** ** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. ** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfullly destroyed and all associated resources are deallocated. +** successfully destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with ** the [sqlite3] object prior to attempting to close the object. ^If ** sqlite3_close() is called on a [database connection] that still has @@ -1015,15 +1022,16 @@ #define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ #define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ /* ** CAPI3REF: Device Characteristics ** -** The xDeviceCapabilities method of the [sqlite3_io_methods] +** The xDeviceCharacteristics method of the [sqlite3_io_methods] ** object returns an integer which is a vector of the these ** bit values expressing I/O characteristics of the mass storage ** device that holds the file that the [sqlite3_io_methods] ** refers to. ** @@ -1075,10 +1083,22 @@ ** sync operation only needs to flush data to mass storage. Inode ** information need not be flushed. If the lower four bits of the flag ** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. ** If the lower four bits equal SQLITE_SYNC_FULL, that means ** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) */ #define SQLITE_SYNC_NORMAL 0x00002 #define SQLITE_SYNC_FULL 0x00003 #define SQLITE_SYNC_DATAONLY 0x00010 @@ -1199,15 +1219,14 @@ int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); int (*xFileControl)(sqlite3_file*, int op, void *pArg); int (*xSectorSize)(sqlite3_file*); int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ - int (*xShmOpen)(sqlite3_file*); + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); - int (*xShmMap)(sqlite3_file*, int iPage, int pgsz, int, void volatile**); void (*xShmBarrier)(sqlite3_file*); - int (*xShmClose)(sqlite3_file*, int deleteFlag); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ /* Additional methods may be added in future releases */ }; /* @@ -1229,16 +1248,27 @@ ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 + /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an @@ -1282,19 +1312,23 @@ ** object once the object has been registered. ** ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** ** The flags argument to xOpen() includes all bits set in ** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] @@ -1301,11 +1335,11 @@ ** or [sqlite3_open16()] is used, then flags includes at least ** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** ** +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application ** that does not care about crash recovery or rollback might make ** the open of a journal file a no-op. Writes to this journal would @@ -1331,39 +1366,40 @@ **
  • [SQLITE_OPEN_DELETEONCLOSE] **
  • [SQLITE_OPEN_EXCLUSIVE] ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the ** SQLITE_OPEN_CREATE, is used to indicate that file should always ** be created, and that it is an error if it already exists. ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that ** the xOpen method must set the sqlite3_file.pMethods to either ** a valid [sqlite3_io_methods] object or to NULL. xOpen must do ** this even if the open fails. SQLite expects that the sqlite3_file.pMethods ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is ** handled as a fatal error by SQLite, vfs implementations should endeavor ** to prevent this by setting mxPathname to a sufficiently large value. @@ -1373,14 +1409,14 @@ ** included in the VFS structure for completeness. ** The xRandomness() function attempts to return nBytes bytes ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or ** greater and the function pointer is not NULL) and will fall back @@ -1426,17 +1462,24 @@ ** the xAccess method of an [sqlite3_vfs] object. They determine ** what kind of permissions the xAccess method is looking for. ** With SQLITE_ACCESS_EXISTS, the xAccess method ** simply checks whether the file exists. ** With SQLITE_ACCESS_READWRITE, the xAccess method -** checks whether the file is both readable and writable. +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. ** With SQLITE_ACCESS_READ, the xAccess method -** checks whether the file is readable. +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. */ #define SQLITE_ACCESS_EXISTS 0 -#define SQLITE_ACCESS_READWRITE 1 -#define SQLITE_ACCESS_READ 2 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ /* ** CAPI3REF: Flags for the xShmLock VFS method ** ** These integer constants define the various locking operations @@ -1766,11 +1809,11 @@ ** statistics. ^(When memory allocation statistics are disabled, the ** following SQLite interfaces become non-operational: ** )^ ** ^Memory allocation statistics are enabled by default unless SQLite is ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory ** allocation statistics are disabled by default. @@ -1780,19 +1823,18 @@ **
    ^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
    ** **
    SQLITE_CONFIG_PAGECACHE
    **
    ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implemenation. @@ -1808,12 +1850,11 @@ ** argument should point to an allocation of at least sz*N bytes of memory. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
    ** **
    SQLITE_CONFIG_HEAP
    **
    ^This option specifies a static memory buffer that SQLite will use @@ -1938,12 +1979,18 @@ ** size of each lookaside buffer slot. ^The third argument is the number of ** slots. The size of the buffer in the first argument must be greater than ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE]
    +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^ ** ** */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -2243,10 +2290,13 @@ */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. ** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -2264,11 +2314,11 @@ ** ** A result table might consist of one or more memory allocations. ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** **
     **        Name        | Age
     **        -----------------------
    @@ -2288,31 +2338,31 @@
     **        azResult[3] = "43";
     **        azResult[4] = "Bob";
     **        azResult[5] = "28";
     **        azResult[6] = "Cindy";
     **        azResult[7] = "21";
    -** 
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 ** string of its 2nd parameter and returns a result table to the ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ const char *zSql, /* SQL to be evaluated */ char ***pazResult, /* Results of the query */ @@ -2460,11 +2510,13 @@ ** by sqlite3_realloc() and the prior allocation is freed. ** ^If sqlite3_realloc() returns NULL, then the prior allocation ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in ** implementation of these routines to be omitted. That capability ** is no longer provided. Only built-in memory allocators can be used. @@ -2703,30 +2755,47 @@ ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time -** of how long that statement took to run. +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. */ SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. ** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** */ @@ -2781,11 +2850,11 @@ ** ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection ** opens in the multi-thread [threading mode] as long as the single-thread ** mode has not been set at compile-time or start-time. ^If the @@ -2906,21 +2975,26 @@ ** ^(This interface allows the size of various constructs to be limited ** on a connection by connection basis. The first parameter is the ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a ** web browser that has its own databases for storing history and ** separate databases controlled by JavaScript applications downloaded @@ -2945,11 +3019,11 @@ ** The synopsis of the meanings of the various limits is shown below. ** Additional information is available at [limits | Limits in SQLite]. ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_COLUMN
    @@ -2963,11 +3037,13 @@ ** ^(
    SQLITE_LIMIT_COMPOUND_SELECT
    **
    The maximum number of terms in a compound SELECT statement.
    )^ ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ ** ** ^(
    SQLITE_LIMIT_ATTACHED
    @@ -2976,12 +3052,11 @@ ** ^(
    SQLITE_LIMIT_LIKE_PATTERN_LENGTH
    **
    The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
    )^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ **
    */ @@ -3049,16 +3124,11 @@ ** **
      **
    1. ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
    2. ** **
    3. ** ^When an error occurs, [sqlite3_step()] will return one of the detailed ** [error codes] or [extended error codes]. ^The legacy behavior was that @@ -3067,15 +3137,20 @@ ** in order to find the underlying cause of the problem. With the "v2" prepare ** interfaces, the underlying reason for the error is returned immediately. **
    4. ** **
    5. -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
    6. **
    */ SQLITE_API int sqlite3_prepare( sqlite3 *db, /* Database handle */ @@ -3113,10 +3188,24 @@ ** SQL text used to create a [prepared statement] if that statement was ** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** the [prepared statement] X is [SELECT] statement and false (zero) if +** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE], +** [ALTER], or [REINDEX] statement. +** If X is a NULL pointer or any other kind of statement, including but +** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE], +** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is +** undefined. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} ** ** SQLite uses the sqlite3_value object to represent all values @@ -3138,11 +3227,11 @@ ** or if SQLite is run in one of reduced mutex modes ** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the ** implementation of [application-defined SQL functions] are protected. ** ^The sqlite3_value object returned by @@ -3184,11 +3273,11 @@ **
  • @VVV **
  • $VVV ** ** ** In the templates above, NNN represents an integer literal, -** and VVV represents an alphanumeric identifer.)^ ^The values of these +** and VVV represents an alphanumeric identifier.)^ ^The values of these ** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** ** ^The first argument to the sqlite3_bind_*() routines is always ** a pointer to the [sqlite3_stmt] object returned from @@ -3212,11 +3301,14 @@ ** ^If the fourth parameter is negative, the length of the string is ** the number of bytes up to the first zero terminator. ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or -** string after SQLite has finished with it. ^If the fifth argument is +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. ** ^If the fifth argument has the value [SQLITE_TRANSIENT], then ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. @@ -3333,10 +3425,12 @@ ** CAPI3REF: Number Of Columns In A Result Set ** ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set @@ -3523,12 +3617,18 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes @@ -3604,22 +3704,30 @@ ** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts ** the string to UTF-8 and then returns the number of bytes. ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object ** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by @@ -3660,14 +3768,14 @@ ** and atof(). SQLite does not really use these functions. It has its ** own equivalent internal routines. The atoi() and atof() names are ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** ** )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** ** )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result ** into the desired format, then invoke sqlite3_column_bytes() or ** sqlite3_column_bytes16() to find the size of the result. Do not mix calls @@ -3729,21 +3837,30 @@ /* ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object @@ -3775,40 +3892,42 @@ ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or ** aggregate takes. ^If this parameter is -1, then the SQL function or ** aggregate may take any number of arguments between 0 and the limit ** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. ** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. @@ -3816,17 +3935,28 @@ ** encoding is used, then the fourth argument should be [SQLITE_ANY]. ** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of ** arguments or differing preferred text encodings. ^SQLite will use ** the implementation that most closely matches the way in which the @@ -3838,15 +3968,10 @@ ** ^A function where the encoding difference is between UTF16le and UTF16be ** is a closer match than a function where the encoding difference is ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. @@ -3868,10 +3993,21 @@ int eTextRep, void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) ); /* ** CAPI3REF: Text Encodings ** @@ -3963,11 +4099,11 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context ** -** Implementions of aggregate SQL functions use this +** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called ** for a particular aggregate function, SQLite ** allocates N of memory, zeroes out that memory, and returns a pointer @@ -4215,73 +4351,106 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. -** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. -** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). -** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +** )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); /* ** CAPI3REF: Collation Needed Callbacks @@ -4366,20 +4535,23 @@ #endif /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -4597,44 +4769,77 @@ ** of heap memory by deallocating non-essential memory allocations ** held by the database library. Memory used to cache database ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. -** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. -** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. -** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. -** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. -*/ -SQLITE_API void sqlite3_soft_heap_limit(int); +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. +** +** ^If the argument N is zero then the soft heap limit is disabled. +** +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: +** +** )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table ** ** ^This routine returns metadata about a specific column of a specific @@ -4754,38 +4959,51 @@ ** it back off again. */ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions -** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. -** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ -** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: +** +**
    +**    int xEntryPoint(
    +**      sqlite3 *db,
    +**      const char **pzErrMsg,
    +**      const struct sqlite3_api_routines *pThunk
    +**    );
    +** 
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); /* ** The interface to the virtual-table mechanism is currently considered @@ -4852,22 +5070,25 @@ /* ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** -** The sqlite3_index_info structure and its substructures is used to +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to ** pass information into and receive the reply from the [xBestIndex] ** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its ** results into the **Outputs** fields. ** ** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -**
    column OP expr
    +**
    column OP expr
    ** ** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is -** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in ** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint ** is usable) and false if it cannot.)^ ** ** ^The optimizer automatically inverts terms of the form "expr OP column" @@ -4923,10 +5144,19 @@ char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ }; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 #define SQLITE_INDEX_CONSTRAINT_LT 16 #define SQLITE_INDEX_CONSTRAINT_GE 32 @@ -4949,11 +5179,13 @@ ** when a new virtual table is be being created or reinitialized. ** ** ^The sqlite3_create_module_v2() interface has a fifth parameter which ** is a pointer to a destructor for the pClientData. ^SQLite will ** invoke the destructor function (if it is not NULL) when SQLite -** no longer needs the pClientData pointer. ^The sqlite3_create_module() +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -5132,10 +5364,34 @@ sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob ); +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + /* ** CAPI3REF: Close A BLOB Handle ** ** ^Closes an open [BLOB handle]. ** @@ -5408,11 +5664,11 @@ ** output variable when querying the system for the current mutex ** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as ** part of system shutdown by the sqlite3_shutdown() function. The ** implementation of this method is expected to release all outstanding @@ -5441,11 +5697,11 @@ ** of passing a NULL pointer instead of a valid mutex handle are undefined ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** ** The xMutexInit() method must be threadsafe. ^It must be harmless to -** invoke xMutexInit() mutiple times within the same process and without +** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** ** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] ** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory @@ -5539,19 +5795,25 @@ ** CAPI3REF: Low-Level Control Of Database Files ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated ** with a particular database identified by the second argument. ^The -** name of the database "main" for the main database or "temp" for the +** name of the database is "main" for the main database or "temp" for the ** TEMP database, or the name that appears after the AS keyword for ** databases that are added using the [ATTACH] SQL command. ** ^A NULL pointer can be used in place of "main" to refer to the ** main database file. ** ^The third and fourth parameters to this routine ** are passed directly through to the second and third parameters of ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. +** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error ** code is not remembered and will not be recalled by [sqlite3_errcode()] ** or [sqlite3_errmsg()]. The underlying xFileControl method might @@ -5605,17 +5867,18 @@ #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information -** about the preformance of SQLite, and optionally to reset various +** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes ** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the @@ -5624,11 +5887,11 @@ ** value. For those parameters ** nothing is written into *pHighwater and the resetFlag is ignored.)^ ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be ** called while other threads are running the same or different SQLite ** interfaces. However the values returned in *pCurrent and @@ -5663,19 +5926,22 @@ ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** +** ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations.
    )^ +** ** ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
    )^ ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
    )^ @@ -5694,11 +5960,11 @@ ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.)^ ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the ** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer ** slots were available. @@ -5724,27 +5990,31 @@ #define SQLITE_STATUS_SCRATCH_OVERFLOW 4 #define SQLITE_STATUS_MALLOC_SIZE 5 #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 #define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status ** ** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that -** determiness the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur ** and the highest instantaneous value is written into *pHiwtr. ^If ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5763,20 +6033,37 @@ **
    ** ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    **
    This parameter returns the number of lookaside memory slots currently ** checked out.
    )^ ** -**
    SQLITE_DBSTATUS_CACHE_USED
    -**
    ^This parameter returns the approximate number of of bytes of heap -** memory used by all pager caches associated with the database connection. +** ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. -** checked out.
    )^ +** +** ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
    **
    */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 #define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_MAX 1 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ /* ** CAPI3REF: Prepared Statement Status ** @@ -5851,122 +6138,134 @@ ** CAPI3REF: Application Defined Page Cache. ** KEYWORDS: {page cache} ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. ** ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After @@ -6138,11 +6437,11 @@ ** ** sqlite3_backup_remaining(), sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source databae file. +** up and the total number of pages in the source database file. ** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces ** retrieve these two values, respectively. ** ** ^The values returned by these functions are only updated by ** sqlite3_backup_step(). ^If the source database is modified during a backup @@ -6234,11 +6533,11 @@ ** ^(There may be at most one unlock-notify callback registered by a ** blocked connection. If sqlite3_unlock_notify() is called when the ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** ** The unlock-notify callback is not reentrant. If an application invokes ** any sqlite3_xxx API functions from within an unlock-notify callback, a @@ -6316,11 +6615,11 @@ /* ** CAPI3REF: String Comparison ** ** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a -** case-indendent fashion, using the same definition of case independence +** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); /* @@ -6440,10 +6739,66 @@ #if 0 } /* End of the 'extern "C"' block */ #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ /************** Include hash.h in the middle of sqliteInt.h ******************/ /************** Begin file hash.h ********************************************/ @@ -7008,10 +7363,11 @@ typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; +typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; typedef struct IndexSample IndexSample; @@ -7114,20 +7470,19 @@ ** following values. ** ** NOTE: These values must match the corresponding PAGER_ values in ** pager.h. */ -#define BTREE_OMIT_JOURNAL 1 /* Do not use journal. No argument */ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ #define BTREE_NO_READLOCK 2 /* Omit readlocks on readonly files */ -#define BTREE_MEMORY 4 /* In-memory DB. No argument */ -#define BTREE_READONLY 8 /* Open the database in read-only mode */ -#define BTREE_READWRITE 16 /* Open for both reading and writing */ -#define BTREE_CREATE 32 /* Create the database if it does not exist */ +#define BTREE_MEMORY 4 /* This is an in-memory DB */ +#define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int); +SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); SQLITE_PRIVATE int sqlite3BtreeSyncDisabled(Btree*); SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); @@ -7155,15 +7510,21 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); /* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR -** of the following flags: +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) */ #define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ -#define BTREE_ZERODATA 2 /* Table has keys only - no data */ -#define BTREE_LEAFDATA 4 /* Data stored in leaves only. Implies INTKEY */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); SQLITE_PRIVATE void sqlite3BtreeTripAllCursors(Btree*, int); @@ -7247,10 +7608,14 @@ #ifdef SQLITE_TEST SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); #endif + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*); +#endif /* ** If we are not using shared cache, then there is no need to ** use mutexes to access the BtShared structures. So make the ** Enter and Leave procedures no-ops. @@ -7377,12 +7742,12 @@ struct SubProgram { VdbeOp *aOp; /* Array of opcodes for sub-program */ int nOp; /* Elements in aOp[] */ int nMem; /* Number of memory cells required */ int nCsr; /* Number of cursors required */ - int nRef; /* Number of pointers to this structure */ void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ }; /* ** A smaller version of VdbeOp used for the VdbeAddOpList() function because ** it takes up less space. @@ -7597,16 +7962,14 @@ #define OP_VColumn 129 #define OP_VNext 131 #define OP_VRename 132 #define OP_VUpdate 133 #define OP_Pagecount 134 -#define OP_Trace 135 -#define OP_Noop 136 -#define OP_Explain 137 - -/* The following opcode values are never used */ -#define OP_NotUsed_138 138 +#define OP_MaxPgcnt 135 +#define OP_Trace 136 +#define OP_Noop 137 +#define OP_Explain 138 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c ** are encoded into bitvectors as follows: @@ -7633,11 +7996,11 @@ /* 88 */ 0x4c, 0x4c, 0x4c, 0x4c, 0x01, 0x24, 0x02, 0x02,\ /* 96 */ 0x00, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00,\ /* 104 */ 0x00, 0x0c, 0x45, 0x15, 0x01, 0x02, 0x00, 0x01,\ /* 112 */ 0x08, 0x05, 0x05, 0x05, 0x00, 0x00, 0x00, 0x02,\ /* 120 */ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x00,\ +/* 128 */ 0x01, 0x00, 0x02, 0x01, 0x00, 0x00, 0x02, 0x02,\ /* 136 */ 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04, 0x04,\ } /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -7664,10 +8027,11 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,int,int,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); #ifdef SQLITE_DEBUG @@ -7681,20 +8045,23 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); -SQLITE_PRIVATE void sqlite3VdbeProgramDelete(sqlite3 *, SubProgram *, int); SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetValue(Vdbe*, int, u8); SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); SQLITE_PRIVATE void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +#endif #ifndef NDEBUG SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); # define VdbeComment(X) sqlite3VdbeComment X @@ -7770,10 +8137,11 @@ ** ** NOTE: These values must match the corresponding BTREE_ values in btree.h. */ #define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ #define PAGER_NO_READLOCK 0x0002 /* Omit readlocks on readonly files */ +#define PAGER_MEMORY 0x0004 /* In-memory database */ /* ** Valid values for the second argument to sqlite3PagerLockingMode(). */ #define PAGER_LOCKINGMODE_QUERY -1 @@ -7810,14 +8178,14 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); -SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u16*, int); +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int); +SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int); SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); @@ -7837,13 +8205,14 @@ SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage*); SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ -SQLITE_PRIVATE int sqlite3PagerPagecount(Pager*, int*); +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); @@ -8303,15 +8672,14 @@ SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); #define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmOpen(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmClose(sqlite3_file *id, int); -SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); /* ** Functions for accessing sqlite3_vfs methods */ SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); @@ -8404,12 +8772,12 @@ #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) #define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK #define sqlite3_mutex_leave(X) -#define sqlite3_mutex_held(X) 1 -#define sqlite3_mutex_notheld(X) 1 +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) #define sqlite3MutexInit() SQLITE_OK #define sqlite3MutexEnd() #endif /* defined(SQLITE_MUTEX_OMIT) */ @@ -8432,18 +8800,10 @@ Schema *pSchema; /* Pointer to database schema (possibly shared) */ }; /* ** An instance of the following structure stores a database schema. -** -** If there are no virtual tables configured in this schema, the -** Schema.db variable is set to NULL. After the first virtual table -** has been added, it is set to point to the database connection -** used to create the connection. Once a virtual table has been -** added to the Schema structure and the Schema.db variable populated, -** only that database connection may use the Schema to prepare -** statements. */ struct Schema { int schema_cookie; /* Database schema version number for this file */ Hash tblHash; /* All tables indexed by name */ Hash idxHash; /* All (named) indices indexed by name */ @@ -8452,13 +8812,10 @@ Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ u8 file_format; /* Schema format version for this file */ u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3 *db; /* "Owner" connection. See comment above */ -#endif }; /* ** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. @@ -8642,10 +8999,11 @@ Savepoint *pSavepoint; /* List of active savepoints */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ + int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER ** mutex, not by sqlite3.mutex. They are used by code in notify.c. ** @@ -8688,17 +9046,18 @@ ** accessing read-only databases */ #define SQLITE_IgnoreChecks 0x00040000 /* Do not enforce check constraints */ #define SQLITE_ReadUncommitted 0x0080000 /* For shared-cache mode */ #define SQLITE_LegacyFileFmt 0x00100000 /* Create new databases in format 1 */ #define SQLITE_FullFSync 0x00200000 /* Use full fsync on the backend */ -#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ +#define SQLITE_CkptFullFSync 0x00400000 /* Use full fsync for checkpoint */ #define SQLITE_RecoveryMode 0x00800000 /* Ignore schema errors */ #define SQLITE_ReverseOrder 0x01000000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x02000000 /* Enable recursive triggers */ #define SQLITE_ForeignKeys 0x04000000 /* Enforce foreign key constraints */ #define SQLITE_AutoIndex 0x08000000 /* Enable automatic indexes */ #define SQLITE_PreferBuiltin 0x10000000 /* Preference to built-in funcs */ +#define SQLITE_LoadExtension 0x20000000 /* Enable load_extension */ /* ** Bits of the sqlite3.flags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface. ** These must be the low-order bits of the flags field. @@ -8707,10 +9066,11 @@ #define SQLITE_ColumnCache 0x02 /* Disable the column cache */ #define SQLITE_IndexSort 0x04 /* Disable indexes for sorting */ #define SQLITE_IndexSearch 0x08 /* Disable indexes for searching */ #define SQLITE_IndexCover 0x10 /* Disable index covering table */ #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* ** Possible values for the sqlite.magic field. ** The numbers are obtained at random and have no special meaning, other @@ -8737,10 +9097,31 @@ void (*xFunc)(sqlite3_context*,int,sqlite3_value**); /* Regular function */ void (*xStep)(sqlite3_context*,int,sqlite3_value**); /* Aggregate step */ void (*xFinalize)(sqlite3_context*); /* Aggregate finalizer */ char *zName; /* SQL name of the function. */ FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ +}; + +/* +** This structure encapsulates a user-function destructor callback (as +** configured using create_function_v2()) and a reference counter. When +** create_function_v2() is called to create a function with a destructor, +** a single object of this type is allocated. FuncDestructor.nRef is set to +** the number of FuncDef objects created (either 1 or 3, depending on whether +** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor +** member of each of the new FuncDef objects is set to point to the allocated +** FuncDestructor. +** +** Thereafter, when one of the FuncDef objects is deleted, the reference +** count on this object is decremented. When it reaches 0, the destructor +** is invoked and the FuncDestructor structure freed. +*/ +struct FuncDestructor { + int nRef; + void (*xDestroy)(void *); + void *pUserData; }; /* ** Possible values for FuncDef.flags */ @@ -8777,19 +9158,19 @@ ** FuncDef.flags variable is set to the value passed as the flags ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0} + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, #zName, 0, 0} #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_UTF8, bNC*SQLITE_FUNC_NEEDCOLL, \ - pArg, 0, xFunc, 0, 0, #zName, 0} + pArg, 0, xFunc, 0, 0, #zName, 0, 0} #define LIKEFUNC(zName, nArg, arg, flags) \ - {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0} + {nArg, SQLITE_UTF8, flags, (void *)arg, 0, likeFunc, 0, 0, #zName, 0, 0} #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8, nc*SQLITE_FUNC_NEEDCOLL, \ - SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0} + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at ** sqlite3.pSavepoint. The first element in the list is the most recently ** opened savepoint. Savepoints are added to the list by the vdbe @@ -8999,17 +9380,17 @@ ** page number. Transient tables are used to hold the results of a ** sub-query that appears instead of a real table name in the FROM clause ** of a SELECT statement. */ struct Table { - sqlite3 *dbMem; /* DB connection used for lookaside allocations. */ char *zName; /* Name of the table or view */ int iPKey; /* If not negative, use aCol[iPKey] as the primary key */ int nCol; /* Number of columns in this table */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ int tnum; /* Root BTree node for this table (see note above) */ + unsigned nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ Select *pSelect; /* NULL for tables. Points to definition if a view. */ u16 nRef; /* Number of pointers to this Table */ u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ FKey *pFKey; /* Linked list of all foreign keys in this table */ @@ -9136,13 +9517,13 @@ ** argument to sqlite3VdbeKeyCompare and is used to control the ** comparison of the two index keys. */ struct KeyInfo { sqlite3 *db; /* The database connection */ - u8 enc; /* Text encoding - one of the TEXT_Utf* values */ + u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ u16 nField; /* Number of entries in aColl[] */ - u8 *aSortOrder; /* If defined an aSortOrder[i] is true, sort DESC */ + u8 *aSortOrder; /* Sort order for each column. May be NULL */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; /* ** An instance of the following structure holds information about a @@ -9574,10 +9955,13 @@ Table *pTab; /* An SQL table corresponding to zName */ Select *pSelect; /* A SELECT statement used in place of a table name */ u8 isPopulated; /* Temporary table associated with SELECT is populated */ u8 jointype; /* Type of join between this able and the previous */ u8 notIndexed; /* True if there is a NOT INDEXED clause */ +#ifndef SQLITE_OMIT_EXPLAIN + u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ +#endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ IdList *pUsing; /* The USING clause of a join */ Bitmask colUsed; /* Bit N (1<" clause */ @@ -9612,10 +9996,11 @@ ** case that more than one of these conditions is true. */ struct WherePlan { u32 wsFlags; /* WHERE_* flags that describe the strategy */ u32 nEq; /* Number of == constraints */ + double nRow; /* Estimated number of rows (for EQP) */ union { Index *pIdx; /* Index when WHERE_INDEXED is true */ struct WhereTerm *pTerm; /* WHERE clause term for OR-search */ sqlite3_index_info *pVtabIdx; /* Virtual table index to use */ } u; @@ -9696,10 +10081,11 @@ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int nLevel; /* Number of nested loop */ struct WhereClause *pWC; /* Decomposition of the WHERE clause */ double savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + double nRowOut; /* Estimated number of output rows */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; /* ** A NameContext defines a context in which to resolve table and column @@ -9771,10 +10157,11 @@ Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + double nSelectRow; /* Estimated number of result rows */ }; /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". @@ -9966,10 +10353,15 @@ Table **apVtabLock; /* Pointer to virtual tables needing locking */ #endif int nHeight; /* Expression tree height of current sub-select */ Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + +#ifndef SQLITE_OMIT_EXPLAIN + int iSelectId; + int iNextSelectId; +#endif }; #ifdef SQLITE_OMIT_VIRTUALTABLE #define IN_DECLARE_VTAB 0 #else @@ -10108,11 +10500,11 @@ char *zText; /* The string collected so far */ int nChar; /* Length of the string so far */ int nAlloc; /* Amount of space allocated in zText */ int mxAlloc; /* Maximum allowed string length */ u8 mallocFailed; /* Becomes true if any memory allocation fails */ - u8 useMalloc; /* True if zText is enlargeable using realloc */ + u8 useMalloc; /* 0: none, 1: sqlite3DbMalloc, 2: sqlite3_malloc */ u8 tooBig; /* Becomes true if string size exceeds limits */ }; /* ** A pointer to this structure is used to communicate information @@ -10260,11 +10652,10 @@ /* ** Internal function prototypes */ SQLITE_PRIVATE int sqlite3StrICmp(const char *, const char *); -SQLITE_PRIVATE int sqlite3IsNumber(const char*, int*, u8); SQLITE_PRIVATE int sqlite3Strlen30(const char*); #define sqlite3StrNICmp sqlite3_strnicmp SQLITE_PRIVATE int sqlite3MallocInit(void); SQLITE_PRIVATE void sqlite3MallocEnd(void); @@ -10284,11 +10675,11 @@ SQLITE_PRIVATE void sqlite3ScratchFree(void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); -SQLITE_PRIVATE int sqlite3MemoryAlarm(void (*)(void*, sqlite3_int64, int), void*, sqlite3_int64); +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); /* ** On systems with ample stack space and that support alloca(), make ** use of alloca() to obtain space for large automatic objects. By default, ** obtain space from malloc(). @@ -10406,11 +10797,11 @@ #else # define sqlite3ViewGetColumnNames(A,B) 0 #endif SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); -SQLITE_PRIVATE void sqlite3DeleteTable(Table*); +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); #else # define sqlite3AutoincrementBegin(X) @@ -10455,11 +10846,10 @@ SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*, int); SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse*,int,int); SQLITE_PRIVATE int sqlite3ExprCode(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeAndCache(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse*, Expr*); @@ -10575,21 +10965,19 @@ # define sqlite3AuthContextPush(a,b,c) # define sqlite3AuthContextPop(a) ((void)(a)) #endif SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); -SQLITE_PRIVATE int sqlite3BtreeFactory(sqlite3 *db, const char *zFilename, - int omitJournal, int nCache, int flags, Btree **ppBtree); SQLITE_PRIVATE int sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); -SQLITE_PRIVATE int sqlite3FitsIn64Bits(const char *, int); +SQLITE_PRIVATE int sqlite3Atoi(const char*); SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE int sqlite3Utf8Read(const u8*, const u8**); /* @@ -10631,20 +11019,21 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(Vdbe *, Index *); SQLITE_PRIVATE void sqlite3TableAffinityStr(Vdbe *, Table *); SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); -SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*); +SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE void sqlite3Error(sqlite3*, int, const char*,...); SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *, Token *); +SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr*, CollSeq*); +SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr*, Token*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); @@ -10661,10 +11050,11 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; +SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte; #endif @@ -10689,11 +11079,11 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); -SQLITE_PRIVATE void sqlite3DeleteIndexSamples(Index*); +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); SQLITE_PRIVATE void sqlite3MinimumFileFormat(Parse*, int, int); SQLITE_PRIVATE void sqlite3SchemaFree(void *); @@ -10700,11 +11090,13 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); SQLITE_PRIVATE KeyInfo *sqlite3IndexKeyinfo(Parse *, Index *); SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*)); + void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + FuncDestructor *pDestructor +); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, char*, int, int); SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum*,const char*,int); @@ -10751,11 +11143,11 @@ # define sqlite3VtabInSync(db) 0 # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) # define sqlite3VtabUnlockList(X) #else -SQLITE_PRIVATE void sqlite3VtabClear(Table*); +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, char **); SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); @@ -10804,13 +11196,13 @@ #define sqlite3FkDropTable(a,b,c) #define sqlite3FkOldmask(a,b) 0 #define sqlite3FkRequired(a,b,c,d) 0 #endif #ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE void sqlite3FkDelete(Table*); +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *, Table*); #else - #define sqlite3FkDelete(a) + #define sqlite3FkDelete(a,b) #endif /* ** Available fault injectors. Should be numbered beginning with 0. @@ -10899,36 +11291,40 @@ ** a single bit set. ** ** sqlite3MemdebugHasType() returns true if any of the bits in its second ** argument match the type set by the previous sqlite3MemdebugSetType(). ** sqlite3MemdebugHasType() is intended for use inside assert() statements. -** For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +** sqlite3MemdebugNoType() returns true if none of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). ** ** Perhaps the most important point is the difference between MEMTYPE_HEAP -** and MEMTYPE_DB. If an allocation is MEMTYPE_DB, that means it might have -** been allocated by lookaside, except the allocation was too large or -** lookaside was already full. It is important to verify that allocations -** that might have been satisfied by lookaside are not passed back to -** non-lookaside free() routines. Asserts such as the example above are -** placed on the non-lookaside free() routines to verify this constraint. +** and MEMTYPE_LOOKASIDE. If an allocation is MEMTYPE_LOOKASIDE, that means +** it might have been allocated by lookaside, except the allocation was +** too large or lookaside was already full. It is important to verify +** that allocations that might have been satisfied by lookaside are not +** passed back to non-lookaside free() routines. Asserts such as the +** example above are placed on the non-lookaside free() routines to verify +** this constraint. ** ** All of this is no-op for a production build. It only comes into ** play when the SQLITE_MEMDEBUG compile-time option is used. */ #ifdef SQLITE_MEMDEBUG SQLITE_PRIVATE void sqlite3MemdebugSetType(void*,u8); SQLITE_PRIVATE int sqlite3MemdebugHasType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); #else # define sqlite3MemdebugSetType(X,Y) /* no-op */ # define sqlite3MemdebugHasType(X,Y) 1 +# define sqlite3MemdebugNoType(X,Y) 1 #endif -#define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_DB 0x02 /* Associated with a database connection */ -#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ -#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_HEAP 0x01 /* General heap allocations */ +#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ +#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ #endif /* _SQLITEINT_H_ */ /************** End of sqliteInt.h *******************************************/ /************** Begin file global.c ******************************************/ @@ -11106,10 +11502,19 @@ ** Hash table for global functions - functions common to all ** database connections. After initialization, this table is ** read-only. */ SQLITE_PRIVATE SQLITE_WSD FuncDefHash sqlite3GlobalFunctions; + +/* +** Constant tokens for values 0 and 1. +*/ +SQLITE_PRIVATE const Token sqlite3IntTokens[] = { + { "0", 1 }, + { "1", 1 } +}; + /* ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts @@ -11546,18 +11951,467 @@ ************************************************************************* ** ** This module implements the sqlite3_status() interface and related ** functionality. */ +/************** Include vdbeInt.h in the middle of status.c ******************/ +/************** Begin file vdbeInt.h *****************************************/ +/* +** 2003 September 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for information that is private to the +** VDBE. This information used to all be at the top of the single +** source code file "vdbe.c". When that file became too big (over +** 6000 lines long) it was split up into several smaller files and +** this header information was factored out. +*/ +#ifndef _VDBEINT_H_ +#define _VDBEINT_H_ + +/* +** SQL is translated into a sequence of instructions to be +** executed by a virtual machine. Each instruction is an instance +** of the following structure. +*/ +typedef struct VdbeOp Op; + +/* +** Boolean values +*/ +typedef unsigned char Bool; + +/* +** A cursor is a pointer into a single BTree within a database file. +** The cursor can seek to a BTree entry with a particular key, or +** loop over all entries of the Btree. You can also insert new BTree +** entries or retrieve the key or data from the entry that the cursor +** is currently pointing to. +** +** Every cursor that the virtual machine has open is represented by an +** instance of the following structure. +** +** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is +** really a single row that represents the NEW or OLD pseudo-table of +** a row trigger. The data for the row is stored in VdbeCursor.pData and +** the rowid is in VdbeCursor.iKey. +*/ +struct VdbeCursor { + BtCursor *pCursor; /* The cursor structure of the backend */ + int iDb; /* Index of cursor database in db->aDb[] (or -1) */ + i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ + Bool zeroed; /* True if zeroed out and ready for reuse */ + Bool rowidIsValid; /* True if lastRowid is valid */ + Bool atFirst; /* True if pointing to first entry */ + Bool useRandomRowid; /* Generate new record numbers semi-randomly */ + Bool nullRow; /* True if pointing to a row with no data */ + Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isTable; /* True if a table requiring integer keys */ + Bool isIndex; /* True if an index containing keys only - no data */ + Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + Btree *pBt; /* Separate file holding temporary table */ + int pseudoTableReg; /* Register holding pseudotable content. */ + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + int nField; /* Number of fields in the header */ + i64 seqCount; /* Sequence counter */ + sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ + const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ + + /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or + ** OP_IsUnique opcode on this cursor. */ + int seekResult; + + /* Cached information about the header for the data record that the + ** cursor is currently pointing to. Only valid if cacheStatus matches + ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of + ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that + ** the cache is out of date. + ** + ** aRow might point to (ephemeral) data for the current row, or it might + ** be NULL. + */ + u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ + int payloadSize; /* Total number of bytes in the record */ + u32 *aType; /* Type values for all entries in the record */ + u32 *aOffset; /* Cached offsets to the start of each columns data */ + u8 *aRow; /* Data for the current row, if all on one page */ +}; +typedef struct VdbeCursor VdbeCursor; + +/* +** When a sub-program is executed (OP_Program), a structure of this type +** is allocated to store the current value of the program counter, as +** well as the current memory cell array and various other frame specific +** values stored in the Vdbe struct. When the sub-program is finished, +** these values are copied back to the Vdbe from the VdbeFrame structure, +** restoring the state of the VM to as it was before the sub-program +** began executing. +** +** The memory for a VdbeFrame object is allocated and managed by a memory +** cell in the parent (calling) frame. When the memory cell is deleted or +** overwritten, the VdbeFrame object is not freed immediately. Instead, it +** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame +** list is deleted when the VM is reset in VdbeHalt(). The reason for doing +** this instead of deleting the VdbeFrame immediately is to avoid recursive +** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the +** child frame are released. +** +** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is +** set to NULL if the currently executing frame is the main program. +*/ +typedef struct VdbeFrame VdbeFrame; +struct VdbeFrame { + Vdbe *v; /* VM this frame belongs to */ + int pc; /* Program Counter in parent (calling) frame */ + Op *aOp; /* Program instructions for parent frame */ + int nOp; /* Size of aOp array */ + Mem *aMem; /* Array of memory cells for parent frame */ + int nMem; /* Number of entries in aMem */ + VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ + u16 nCursor; /* Number of entries in apCsr */ + void *token; /* Copy of SubProgram.token */ + int nChildMem; /* Number of memory cells for child frame */ + int nChildCsr; /* Number of cursors for child frame */ + i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + int nChange; /* Statement changes (Vdbe.nChanges) */ + VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ +}; + +#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) + +/* +** A value for VdbeCursor.cacheValid that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + +/* +** Internally, the vdbe manipulates nearly all SQL values as Mem +** structures. Each Mem struct may cache multiple representations (string, +** integer etc.) of the same value. A value (and therefore Mem structure) +** has the following properties: +** +** Each value has a manifest type. The manifest type of the value stored +** in a Mem struct is returned by the MemType(Mem*) macro. The type is +** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or +** SQLITE_BLOB. +*/ +struct Mem { + union { + i64 i; /* Integer value. */ + int nZero; /* Used when bit MEM_Zero is set in flags */ + FuncDef *pDef; /* Used only when flags==MEM_Agg */ + RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ + VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ + } u; + double r; /* Real value */ + sqlite3 *db; /* The associated database connection */ + char *z; /* String or BLOB value */ + int n; /* Number of characters in string value, excluding '\0' */ + u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ + u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ + u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ +#ifdef SQLITE_DEBUG + Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ + void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ +#endif + void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ + char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ +}; + +/* One or more of the following flags are set to indicate the validOK +** representations of the value stored in the Mem struct. +** +** If the MEM_Null flag is set, then the value is an SQL NULL value. +** No other flags may be set in this case. +** +** If the MEM_Str flag is set then Mem.z points at a string representation. +** Usually this is encoded in the same unicode encoding as the main +** database (see below for exceptions). If the MEM_Term flag is also +** set, then the string is nul terminated. The MEM_Int and MEM_Real +** flags may coexist with the MEM_Str flag. +** +** Multiple of these values can appear in Mem.flags. But only one +** at a time can appear in Mem.type. +*/ +#define MEM_Null 0x0001 /* Value is NULL */ +#define MEM_Str 0x0002 /* Value is a string */ +#define MEM_Int 0x0004 /* Value is an integer */ +#define MEM_Real 0x0008 /* Value is a real number */ +#define MEM_Blob 0x0010 /* Value is a BLOB */ +#define MEM_RowSet 0x0020 /* Value is a RowSet object */ +#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +#define MEM_Invalid 0x0080 /* Value is undefined */ +#define MEM_TypeMask 0x00ff /* Mask of type bits */ + +/* Whenever Mem contains a valid string or blob representation, one of +** the following flags must be set to determine the memory management +** policy for Mem.z. The MEM_Term flag tells us whether or not the +** string is \000 or \u0000 terminated +*/ +#define MEM_Term 0x0200 /* String rep is nul terminated */ +#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ +#define MEM_Static 0x0800 /* Mem.z points to a static string */ +#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ +#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#ifdef SQLITE_OMIT_INCRBLOB + #undef MEM_Zero + #define MEM_Zero 0x0000 +#endif + +/* +** Clear any existing type flags from a Mem and replace them with f +*/ +#define MemSetTypeFlag(p, f) \ + ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) + +/* +** Return true if a memory cell is not marked as invalid. This macro +** is for use inside assert() statements only. +*/ +#ifdef SQLITE_DEBUG +#define memIsValid(M) ((M)->flags & MEM_Invalid)==0 +#endif + + +/* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains +** additional information about auxiliary information bound to arguments +** of the function. This is used to implement the sqlite3_get_auxdata() +** and sqlite3_set_auxdata() APIs. The "auxdata" is some auxiliary data +** that can be associated with a constant argument to a function. This +** allows functions such as "regexp" to compile their constant regular +** expression argument once and reused the compiled code for multiple +** invocations. +*/ +struct VdbeFunc { + FuncDef *pFunc; /* The definition of the function */ + int nAux; /* Number of entries allocated for apAux[] */ + struct AuxData { + void *pAux; /* Aux data for the i-th argument */ + void (*xDelete)(void *); /* Destructor for the aux data */ + } apAux[1]; /* One slot for each function argument */ +}; + +/* +** The "context" argument for a installable function. A pointer to an +** instance of this structure is the first argument to the routines used +** implement the SQL functions. +** +** There is a typedef for this structure in sqlite.h. So all routines, +** even the public interface to SQLite, can use a pointer to this structure. +** But this file is the only place where the internal details of this +** structure are known. +** +** This structure is defined inside of vdbeInt.h because it uses substructures +** (Mem) which are only defined there. +*/ +struct sqlite3_context { + FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ + VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ + Mem s; /* The return value is stored here */ + Mem *pMem; /* Memory cell used to store aggregate context */ + int isError; /* Error code returned by the function. */ + CollSeq *pColl; /* Collating sequence */ +}; + +/* +** A Set structure is used for quick testing to see if a value +** is part of a small set. Sets are used to implement code like +** this: +** x.y IN ('hi','hoo','hum') +*/ +typedef struct Set Set; +struct Set { + Hash hash; /* A set is just a hash table */ + HashElem *prev; /* Previously accessed hash elemen */ +}; + +/* +** An instance of the virtual machine. This structure contains the complete +** state of the virtual machine. +** +** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile() +** is really a pointer to an instance of this structure. +** +** The Vdbe.inVtabMethod variable is set to non-zero for the duration of +** any virtual table method invocations made by the vdbe program. It is +** set to 2 for xDestroy method calls and 1 for all other methods. This +** variable is used for two purposes: to allow xDestroy methods to execute +** "DROP TABLE" statements and to prevent some nasty side effects of +** malloc failure when SQLite is invoked recursively by a virtual table +** method function. +*/ +struct Vdbe { + sqlite3 *db; /* The database connection that owns this statement */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Number of slots allocated for aOp[] */ + Op *aOp; /* Space to hold the virtual machine's program */ + int nLabel; /* Number of labels used */ + int nLabelAlloc; /* Number of slots allocated in aLabel[] */ + int *aLabel; /* Space to hold the labels */ + Mem **apArg; /* Arguments to currently executing user function */ + Mem *aColName; /* Column names to return */ + Mem *pResultSet; /* Pointer to an array of results */ + u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nCursor; /* Number of slots in apCsr[] */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 okVar; /* True if azVar[] has been initialized */ + ynVar nVar; /* Number of entries in aVar[] */ + Mem *aVar; /* Values for the OP_Variable opcode. */ + char **azVar; /* Name of variables */ + u32 magic; /* Magic number for sanity checking */ + int nMem; /* Number of memory locations currently allocated */ + Mem *aMem; /* The memory locations */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ + char *zErrMsg; /* Error message written here */ + u8 explain; /* True if EXPLAIN present on SQL command */ + u8 changeCntOn; /* True to update the change-counter */ + u8 expired; /* True if the VM needs to be recompiled */ + u8 runOnlyOnce; /* Automatically expire on reset */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + u8 inVtabMethod; /* See comments above */ + u8 usesStmtJournal; /* True if uses a statement journal */ + u8 readOnly; /* True for read-only statements */ + u8 isPrepareV2; /* True if prepared with prepare_v2() */ + int nChange; /* Number of db changes made since last reset */ + int btreeMask; /* Bitmask of db->aDb[] entries referenced */ + i64 startTime; /* Time when query started - used for profiling */ + BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */ + int aCounter[3]; /* Counters used by sqlite3_stmt_status() */ + char *zSql; /* Text of the SQL statement that generated this */ + void *pFree; /* Free this when deleting the vdbe */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + int iStatement; /* Statement number (or 0 if has not opened stmt) */ +#ifdef SQLITE_DEBUG + FILE *trace; /* Write an execution trace here, if not NULL */ +#endif + VdbeFrame *pFrame; /* Parent frame */ + VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ + int nFrame; /* Number of frames in pFrame list */ + u32 expmask; /* Binding to these vars invalidates VM */ + SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ +}; + +/* +** The following are allowed values for Vdbe.magic +*/ +#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ + +/* +** Function prototypes +*/ +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); +#endif +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, int, Mem*, int); +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc*, int); + +int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); +#ifdef SQLITE_OMIT_FLOATING_POINT +# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 +#else +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); +#endif +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); +SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); +SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe*,Mem*); +#endif + +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +#else +# define sqlite3VdbeCheckFk(p,i) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p); +#else +# define sqlite3VdbeMutexArrayEnter(p) +#endif + +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); +#else + #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK +#endif + +#endif /* !defined(_VDBEINT_H_) */ + +/************** End of vdbeInt.h *********************************************/ +/************** Continuing where we left off in status.c *********************/ /* ** Variables in which to record status information. */ typedef struct sqlite3StatType sqlite3StatType; static SQLITE_WSD struct sqlite3StatType { - int nowValue[9]; /* Current value */ - int mxValue[9]; /* Maximum value */ + int nowValue[10]; /* Current value */ + int mxValue[10]; /* Maximum value */ } sqlite3Stat = { {0,}, {0,} }; /* The "wsdStat" macro will resolve to the status information ** state vector. If writable static data is unsupported on the target, @@ -11635,10 +12489,12 @@ int op, /* Status verb */ int *pCurrent, /* Write current value here */ int *pHighwater, /* Write high-water mark here */ int resetFlag /* Reset high-water mark if true */ ){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { *pCurrent = db->lookaside.nOut; *pHighwater = db->lookaside.mxOut; if( resetFlag ){ @@ -11653,26 +12509,92 @@ ** highwater mark is meaningless and is returned as zero. */ case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; + sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ Pager *pPager = sqlite3BtreePager(pBt); totalUsed += sqlite3PagerMemUsed(pPager); } } + sqlite3BtreeLeaveAll(db); *pCurrent = totalUsed; *pHighwater = 0; break; } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store the schema for all databases (main, temp, and any ATTACHed + ** databases. *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_SCHEMA_USED: { + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( ALWAYS(pSchema!=0) ){ + HashElem *p; + + nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( + pSchema->tblHash.count + + pSchema->trigHash.count + + pSchema->idxHash.count + + pSchema->fkeyHash.count + ); + nByte += sqlite3MallocSize(pSchema->tblHash.ht); + nByte += sqlite3MallocSize(pSchema->trigHash.ht); + nByte += sqlite3MallocSize(pSchema->idxHash.ht); + nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + + for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); + } + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTable(db, (Table *)sqliteHashData(p)); + } + } + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store all prepared statements. + ** *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_STMT_USED: { + struct Vdbe *pVdbe; /* Used to iterate through VMs */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + sqlite3VdbeDeleteObject(db, pVdbe); + } + db->pnBytesFreed = 0; + + *pHighwater = 0; + *pCurrent = nByte; + + break; + } + default: { - return SQLITE_ERROR; + rc = SQLITE_ERROR; } } - return SQLITE_OK; + sqlite3_mutex_leave(db->mutex); + return rc; } /************** End of status.c **********************************************/ /************** Begin file date.c ********************************************/ /* @@ -11805,16 +12727,10 @@ end_getDigits: va_end(ap); return cnt; } -/* -** Read text from z[] and convert into a floating point number. Return -** the number of digits converted. -*/ -#define getValue sqlite3AtoF - /* ** Parse a timezone extension on the end of a date-time. ** The extension is of the form: ** ** (+/-)HH:MM @@ -12012,21 +12928,19 @@ static int parseDateOrTime( sqlite3_context *context, const char *zDate, DateTime *p ){ - int isRealNum; /* Return from sqlite3IsNumber(). Not used */ + double r; if( parseYyyyMmDd(zDate,p)==0 ){ return 0; }else if( parseHhMmSs(zDate, p)==0 ){ return 0; }else if( sqlite3StrICmp(zDate,"now")==0){ setDateTimeToCurrent(context, p); return 0; - }else if( sqlite3IsNumber(zDate, &isRealNum, SQLITE_UTF8) ){ - double r; - getValue(zDate, &r); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); p->validJD = 1; return 0; } return 1; @@ -12243,12 +13157,13 @@ ** ** Move the date to the same time on the next occurrence of ** weekday N where 0==Sunday, 1==Monday, and so forth. If the ** date is already on the appropriate weekday, this is a no-op. */ - if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0 - && (n=(int)r)==r && n>=0 && r<7 ){ + if( strncmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p); p->validTZ = 0; p->validJD = 0; computeJD(p); @@ -12299,12 +13214,15 @@ case '6': case '7': case '8': case '9': { double rRounder; - n = getValue(z, &r); - assert( n>=1 ); + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + rc = 1; + break; + } if( z[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be ** omitted. @@ -12871,30 +13789,27 @@ return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); } SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ return id->pMethods->xDeviceCharacteristics(id); } -SQLITE_PRIVATE int sqlite3OsShmOpen(sqlite3_file *id){ - return id->pMethods->xShmOpen(id); -} SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ return id->pMethods->xShmLock(id, offset, n, flags); } SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id){ id->pMethods->xShmBarrier(id); } -SQLITE_PRIVATE int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){ - return id->pMethods->xShmClose(id, deleteFlag); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int deleteFlag){ + return id->pMethods->xShmUnmap(id, deleteFlag); } SQLITE_PRIVATE int sqlite3OsShmMap( - sqlite3_file *id, - int iPage, - int pgsz, - int isWrite, - void volatile **pp + sqlite3_file *id, /* Database file handle */ + int iPage, + int pgsz, + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Pointer to mapping */ ){ - return id->pMethods->xShmMap(id, iPage, pgsz, isWrite, pp); + return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } /* ** The next group of routines are convenience wrappers around the ** VFS methods. @@ -12906,15 +13821,15 @@ int flags, int *pFlagsOut ){ int rc; DO_OS_MALLOC_TEST(0); - /* 0x7f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed + /* 0x87f3f is a mask of SQLITE_OPEN_ flags that are valid to be passed ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x7f3f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f3f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ return pVfs->xDelete(pVfs, zPath, dirSync); @@ -12957,10 +13872,16 @@ SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); } SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); }else{ double r; rc = pVfs->xCurrentTime(pVfs, &r); @@ -13340,11 +14261,11 @@ ** routines and redirected to xFree. */ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); - nByte = ROUND8(nByte); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ p--; p = realloc(p, nByte+8 ); if( p ){ p[0] = nByte; p++; @@ -13746,10 +14667,11 @@ */ static void *sqlite3MemRealloc(void *pPrior, int nByte){ struct MemBlockHdr *pOldHdr; void *pNew; assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ pOldHdr = sqlite3MemsysGetHeader(pPrior); pNew = sqlite3MemMalloc(nByte); if( pNew ){ memcpy(pNew, pPrior, nByteiSize ? nByte : pOldHdr->iSize); if( nByte>pOldHdr->iSize ){ @@ -13780,11 +14702,11 @@ /* ** Set the "type" of an allocation. */ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); pHdr->eType = eType; } @@ -13799,27 +14721,42 @@ ** ** assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); */ SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; - if( p ){ + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)==0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ - assert( (pHdr->eType & (pHdr->eType-1))==0 ); /* Only one type bit set */ - if( (pHdr->eType&eType)==0 ){ - void **pBt; - pBt = (void**)pHdr; - pBt -= pHdr->nBacktraceSlots; - backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(stderr)); - fprintf(stderr, "\n"); + if( (pHdr->eType&eType)!=0 ){ rc = 0; } } return rc; } - /* ** Set the number of backtrace levels kept for each allocation. ** A value of zero turns off backtracing. The number is always rounded ** up to a multiple of 2. @@ -15000,11 +15937,11 @@ */ static void *memsys5Realloc(void *pPrior, int nBytes){ int nOld; void *p; assert( pPrior!=0 ); - assert( (nBytes&(nBytes-1))==0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ assert( nBytes>=0 ); if( nBytes==0 ){ return 0; } nOld = memsys5Size(pPrior); @@ -16058,10 +16995,11 @@ #else /* Use the built-in recursive mutexes if they are available. */ pthread_mutex_lock(&p->mutex); #if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); p->owner = pthread_self(); p->nRef++; #endif #endif @@ -16130,10 +17068,11 @@ */ static void pthreadMutexLeave(sqlite3_mutex *p){ assert( pthreadMutexHeld(p) ); #if SQLITE_MUTEX_NREF p->nRef--; + if( p->nRef==0 ) p->owner = 0; #endif assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX if( p->nRef==0 ){ @@ -16394,11 +17333,11 @@ ** allocated mutex. SQLite is careful to deallocate every ** mutex that it allocates. */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); - assert( p->nRef==0 ); + assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -16418,10 +17357,11 @@ DWORD tid = GetCurrentThreadId(); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); #endif EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG + assert( p->nRef>0 || p->owner==0 ); p->owner = tid; p->nRef++; if( p->trace ){ printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); } @@ -16471,10 +17411,11 @@ #ifndef NDEBUG DWORD tid = GetCurrentThreadId(); assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; + if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ @@ -16520,10 +17461,70 @@ ************************************************************************* ** ** Memory allocation functions used throughout sqlite. */ +/* +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. +*/ +SQLITE_API int sqlite3_release_memory(int n){ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + return sqlite3PcacheReleaseMemory(n); +#else + /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine + ** is a no-op returning zero if SQLite is not compiled with + ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ + UNUSED_PARAMETER(n); + return 0; +#endif +} + +/* +** An instance of the following object records the location of +** each unused scratch buffer. +*/ +typedef struct ScratchFreeslot { + struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ +} ScratchFreeslot; + +/* +** State information local to the memory allocation subsystem. +*/ +static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + + /* + ** The alarm callback and its arguments. The mem0.mutex lock will + ** be held while the callback is running. Recursive calls into + ** the memory subsystem are allowed, but no new callbacks will be + ** issued. + */ + sqlite3_int64 alarmThreshold; + void (*alarmCallback)(void*, sqlite3_int64,int); + void *alarmArg; + + /* + ** Pointers to the end of sqlite3GlobalConfig.pScratch memory + ** (so that a range test can be used to determine if an allocation + ** being freed came from pScratch) and a pointer to the list of + ** unused scratch allocations. + */ + void *pScratchEnd; + ScratchFreeslot *pScratchFree; + u32 nScratchFree; + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; + +#define mem0 GLOBAL(struct Mem0Global, mem0) + /* ** This routine runs when the memory allocator sees that the ** total memory allocation is about to exceed the soft heap ** limit. */ @@ -16534,82 +17535,70 @@ ){ UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_release_memory(allocSize); } +/* +** Change the alarm callback +*/ +static int sqlite3MemoryAlarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + int nUsed; + sqlite3_mutex_enter(mem0.mutex); + mem0.alarmCallback = xCallback; + mem0.alarmArg = pArg; + mem0.alarmThreshold = iThreshold; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + mem0.nearlyFull = (iThreshold>0 && iThreshold<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3MemoryAlarm. +*/ +SQLITE_API int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + return sqlite3MemoryAlarm(xCallback, pArg, iThreshold); +} +#endif + /* ** Set the soft heap-size limit for the library. Passing a zero or ** negative value indicates no limit. */ -SQLITE_API void sqlite3_soft_heap_limit(int n){ - sqlite3_uint64 iLimit; - int overage; - if( n<0 ){ - iLimit = 0; - }else{ - iLimit = n; - } +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; + sqlite3_int64 excess; #ifndef SQLITE_OMIT_AUTOINIT sqlite3_initialize(); #endif - if( iLimit>0 ){ - sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, iLimit); + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.alarmThreshold; + sqlite3_mutex_leave(mem0.mutex); + if( n<0 ) return priorLimit; + if( n>0 ){ + sqlite3MemoryAlarm(softHeapLimitEnforcer, 0, n); }else{ sqlite3MemoryAlarm(0, 0, 0); } - overage = (int)(sqlite3_memory_used() - (i64)n); - if( overage>0 ){ - sqlite3_release_memory(overage); - } -} - -/* -** Attempt to release up to n bytes of non-essential memory currently -** held by SQLite. An example of non-essential memory is memory used to -** cache database pages that are not currently in use. -*/ -SQLITE_API int sqlite3_release_memory(int n){ -#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT - int nRet = 0; - nRet += sqlite3PcacheReleaseMemory(n-nRet); - return nRet; -#else - UNUSED_PARAMETER(n); - return SQLITE_OK; -#endif -} - -/* -** State information local to the memory allocation subsystem. -*/ -static SQLITE_WSD struct Mem0Global { - /* Number of free pages for scratch and page-cache memory */ - u32 nScratchFree; - u32 nPageFree; - - sqlite3_mutex *mutex; /* Mutex to serialize access */ - - /* - ** The alarm callback and its arguments. The mem0.mutex lock will - ** be held while the callback is running. Recursive calls into - ** the memory subsystem are allowed, but no new callbacks will be - ** issued. - */ - sqlite3_int64 alarmThreshold; - void (*alarmCallback)(void*, sqlite3_int64,int); - void *alarmArg; - - /* - ** Pointers to the end of sqlite3GlobalConfig.pScratch and - ** sqlite3GlobalConfig.pPage to a block of memory that records - ** which pages are available. - */ - u32 *aScratchFree; - u32 *aPageFree; -} mem0 = { 0, 0, 0, 0, 0, 0, 0, 0 }; - -#define mem0 GLOBAL(struct Mem0Global, mem0) + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; +} +SQLITE_API void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); +} /* ** Initialize the memory allocation subsystem. */ SQLITE_PRIVATE int sqlite3MallocInit(void){ @@ -16619,39 +17608,48 @@ memset(&mem0, 0, sizeof(mem0)); if( sqlite3GlobalConfig.bCoreMutex ){ mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); } if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 - && sqlite3GlobalConfig.nScratch>=0 ){ - int i; - sqlite3GlobalConfig.szScratch = ROUNDDOWN8(sqlite3GlobalConfig.szScratch-4); - mem0.aScratchFree = (u32*)&((char*)sqlite3GlobalConfig.pScratch) - [sqlite3GlobalConfig.szScratch*sqlite3GlobalConfig.nScratch]; - for(i=0; i0 ){ + int i, n, sz; + ScratchFreeslot *pSlot; + sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); + sqlite3GlobalConfig.szScratch = sz; + pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; + n = sqlite3GlobalConfig.nScratch; + mem0.pScratchFree = pSlot; + mem0.nScratchFree = n; + for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); + pSlot = pSlot->pNext; + } + pSlot->pNext = 0; + mem0.pScratchEnd = (void*)&pSlot[1]; }else{ + mem0.pScratchEnd = 0; sqlite3GlobalConfig.pScratch = 0; sqlite3GlobalConfig.szScratch = 0; - } - if( sqlite3GlobalConfig.pPage && sqlite3GlobalConfig.szPage>=512 - && sqlite3GlobalConfig.nPage>=1 ){ - int i; - int overhead; - int sz = ROUNDDOWN8(sqlite3GlobalConfig.szPage); - int n = sqlite3GlobalConfig.nPage; - overhead = (4*n + sz - 1)/sz; - sqlite3GlobalConfig.nPage -= overhead; - mem0.aPageFree = (u32*)&((char*)sqlite3GlobalConfig.pPage) - [sqlite3GlobalConfig.szPage*sqlite3GlobalConfig.nPage]; - for(i=0; i= mem0.alarmThreshold ){ + mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); + }else{ + mem0.nearlyFull = 0; } } p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( p==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nFull); p = sqlite3GlobalConfig.m.xMalloc(nFull); } +#endif if( p ){ nFull = sqlite3MallocSize(p); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; return nFull; } @@ -16767,11 +17741,13 @@ ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ SQLITE_PRIVATE void *sqlite3Malloc(int n){ void *p; - if( n<=0 || n>=0x7fffff00 ){ + if( n<=0 /* IMP: R-65312-04917 */ + || n>=0x7fffff00 + ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving ** 255 bytes of overhead. SQLite itself will never use anything near ** this amount. The only way to reach the limit is with sqlite3_malloc() */ @@ -16781,10 +17757,11 @@ mallocWithAlarm(n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ p = sqlite3GlobalConfig.m.xMalloc(n); } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ return p; } /* ** This version of the memory allocation is for use by the application. @@ -16818,106 +17795,94 @@ ** embedded processor. */ SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); + + sqlite3_mutex_enter(mem0.mutex); + if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ + p = mem0.pScratchFree; + mem0.pScratchFree = mem0.pScratchFree->pNext; + mem0.nScratchFree--; + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + n = mallocWithAlarm(n, &p); + if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3GlobalConfig.m.xMalloc(n); + } + sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); + } + assert( sqlite3_mutex_notheld(mem0.mutex) ); + #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocation per thread - ** is outstanding at one time. (This is only checked in the + /* Verify that no more than two scratch allocations per thread + ** are outstanding at one time. (This is only checked in the ** single-threaded case since checking in the multi-threaded case ** would be much more complicated.) */ assert( scratchAllocOut<=1 ); -#endif - - if( sqlite3GlobalConfig.szScratch=(void*)mem0.aScratchFree ){ - assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - if( sqlite3GlobalConfig.bMemstat ){ - int iSize = sqlite3MallocSize(p); - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); - sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); - sqlite3GlobalConfig.m.xFree(p); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3GlobalConfig.m.xFree(p); - } - }else{ - int i; - i = (int)((u8*)p - (u8*)sqlite3GlobalConfig.pScratch); - i /= sqlite3GlobalConfig.szScratch; - assert( i>=0 && i=1 && scratchAllocOut<=2 ); - scratchAllocOut = 0; + scratchAllocOut--; #endif + if( p>=sqlite3GlobalConfig.pScratch && ppNext = mem0.pScratchFree; + mem0.pScratchFree = pSlot; + mem0.nScratchFree++; + assert( mem0.nScratchFree<=sqlite3GlobalConfig.nScratch ); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, -1); + sqlite3_mutex_leave(mem0.mutex); + }else{ + /* Release memory back to the heap */ + assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_SCRATCH) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + if( sqlite3GlobalConfig.bMemstat ){ + int iSize = sqlite3MallocSize(p); + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -iSize); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } } } } /* ** TRUE if p is a lookaside memory allocation from db */ #ifndef SQLITE_OMIT_LOOKASIDE static int isLookaside(sqlite3 *db, void *p){ - return db && p && p>=db->lookaside.pStart && plookaside.pEnd; + return p && p>=db->lookaside.pStart && plookaside.pEnd; } #else #define isLookaside(A,B) 0 #endif @@ -16925,32 +17890,36 @@ ** Return the size of a memory allocation previously obtained from ** sqlite3Malloc() or sqlite3_malloc(). */ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ + if( db && isLookaside(db, p) ){ return db->lookaside.sz; }else{ - assert( sqlite3MemdebugHasType(p, - db ? (MEMTYPE_DB|MEMTYPE_HEAP) : MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); return sqlite3GlobalConfig.m.xSize(p); } } /* ** Free memory previously obtained from sqlite3Malloc(). */ SQLITE_API void sqlite3_free(void *p){ - if( p==0 ) return; + if( p==0 ) return; /* IMP: R-49053-54554 */ + assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); + sqlite3StatusAdd(SQLITE_STATUS_MALLOC_COUNT, -1); sqlite3GlobalConfig.m.xFree(p); sqlite3_mutex_leave(mem0.mutex); }else{ sqlite3GlobalConfig.m.xFree(p); } @@ -16960,40 +17929,51 @@ ** Free memory that might be associated with a particular database ** connection. */ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - db->lookaside.nOut--; - }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - sqlite3_free(p); - } + if( db ){ + if( db->pnBytesFreed ){ + *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + return; + } + if( isLookaside(db, p) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + db->lookaside.nOut--; + return; + } + } + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); } /* ** Change the size of an existing memory allocation */ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, int nBytes){ int nOld, nNew; void *pNew; if( pOld==0 ){ - return sqlite3Malloc(nBytes); + return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ } if( nBytes<=0 ){ - sqlite3_free(pOld); + sqlite3_free(pOld); /* IMP: R-31593-10574 */ return 0; } if( nBytes>=0x7fffff00 ){ /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ return 0; } nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); @@ -17001,10 +17981,11 @@ if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nNew-nOld >= mem0.alarmThreshold ){ sqlite3MallocAlarm(nNew-nOld); } assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ sqlite3MallocAlarm(nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } @@ -17014,10 +17995,11 @@ } sqlite3_mutex_leave(mem0.mutex); }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ return pNew; } /* ** The public interface to sqlite3Realloc. Make sure that the memory @@ -17073,10 +18055,11 @@ ** that all prior mallocs (ex: "a") worked too. */ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, int n){ void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( db==0 || db->pnBytesFreed==0 ); #ifndef SQLITE_OMIT_LOOKASIDE if( db ){ LookasideSlot *pBuf; if( db->mallocFailed ){ return 0; @@ -17098,12 +18081,12 @@ #endif p = sqlite3Malloc(n); if( !p && db ){ db->mallocFailed = 1; } - sqlite3MemdebugSetType(p, - (db && db->lookaside.bEnabled) ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(p, MEMTYPE_DB | + ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); return p; } /* ** Resize the block of memory pointed to by p to n bytes. If the @@ -17125,18 +18108,20 @@ if( pNew ){ memcpy(pNew, p, db->lookaside.sz); sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc(p, n); if( !pNew ){ + sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; } - sqlite3MemdebugSetType(pNew, - db->lookaside.bEnabled ? MEMTYPE_DB : MEMTYPE_HEAP); + sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } return pNew; } @@ -18005,11 +18990,15 @@ p->tooBig = 1; return; }else{ p->nAlloc = (int)szNew; } - zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + if( p->useMalloc==1 ){ + zNew = sqlite3DbMallocRaw(p->db, p->nAlloc ); + }else{ + zNew = sqlite3_malloc(p->nAlloc); + } if( zNew ){ memcpy(zNew, p->zText, p->nChar); sqlite3StrAccumReset(p); p->zText = zNew; }else{ @@ -18030,11 +19019,15 @@ */ SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ p->zText[p->nChar] = 0; if( p->useMalloc && p->zText==p->zBase ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( p->useMalloc==1 ){ + p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + }else{ + p->zText = sqlite3_malloc(p->nChar+1); + } if( p->zText ){ memcpy(p->zText, p->zBase, p->nChar+1); }else{ p->mallocFailed = 1; } @@ -18046,11 +19039,15 @@ /* ** Reset an StrAccum string. Reclaim all malloced memory. */ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ if( p->zText!=p->zBase ){ - sqlite3DbFree(p->db, p->zText); + if( p->useMalloc==1 ){ + sqlite3DbFree(p->db, p->zText); + }else{ + sqlite3_free(p->zText); + } } p->zText = 0; } /* @@ -18128,10 +19125,11 @@ StrAccum acc; #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + acc.useMalloc = 2; sqlite3VXPrintf(&acc, 0, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -18421,433 +19419,10 @@ ** BOM or Byte Order Mark: ** 0xff 0xfe little-endian utf-16 follows ** 0xfe 0xff big-endian utf-16 follows ** */ -/************** Include vdbeInt.h in the middle of utf.c *********************/ -/************** Begin file vdbeInt.h *****************************************/ -/* -** 2003 September 6 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for information that is private to the -** VDBE. This information used to all be at the top of the single -** source code file "vdbe.c". When that file became too big (over -** 6000 lines long) it was split up into several smaller files and -** this header information was factored out. -*/ -#ifndef _VDBEINT_H_ -#define _VDBEINT_H_ - -/* -** SQL is translated into a sequence of instructions to be -** executed by a virtual machine. Each instruction is an instance -** of the following structure. -*/ -typedef struct VdbeOp Op; - -/* -** Boolean values -*/ -typedef unsigned char Bool; - -/* -** A cursor is a pointer into a single BTree within a database file. -** The cursor can seek to a BTree entry with a particular key, or -** loop over all entries of the Btree. You can also insert new BTree -** entries or retrieve the key or data from the entry that the cursor -** is currently pointing to. -** -** Every cursor that the virtual machine has open is represented by an -** instance of the following structure. -** -** If the VdbeCursor.isTriggerRow flag is set it means that this cursor is -** really a single row that represents the NEW or OLD pseudo-table of -** a row trigger. The data for the row is stored in VdbeCursor.pData and -** the rowid is in VdbeCursor.iKey. -*/ -struct VdbeCursor { - BtCursor *pCursor; /* The cursor structure of the backend */ - int iDb; /* Index of cursor database in db->aDb[] (or -1) */ - i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - Bool zeroed; /* True if zeroed out and ready for reuse */ - Bool rowidIsValid; /* True if lastRowid is valid */ - Bool atFirst; /* True if pointing to first entry */ - Bool useRandomRowid; /* Generate new record numbers semi-randomly */ - Bool nullRow; /* True if pointing to a row with no data */ - Bool deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ - Bool isTable; /* True if a table requiring integer keys */ - Bool isIndex; /* True if an index containing keys only - no data */ - i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - Btree *pBt; /* Separate file holding temporary table */ - int pseudoTableReg; /* Register holding pseudotable content. */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int nField; /* Number of fields in the header */ - i64 seqCount; /* Sequence counter */ - sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ - const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ - - /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or - ** OP_IsUnique opcode on this cursor. */ - int seekResult; - - /* Cached information about the header for the data record that the - ** cursor is currently pointing to. Only valid if cacheStatus matches - ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of - ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that - ** the cache is out of date. - ** - ** aRow might point to (ephemeral) data for the current row, or it might - ** be NULL. - */ - u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ - int payloadSize; /* Total number of bytes in the record */ - u32 *aType; /* Type values for all entries in the record */ - u32 *aOffset; /* Cached offsets to the start of each columns data */ - u8 *aRow; /* Data for the current row, if all on one page */ -}; -typedef struct VdbeCursor VdbeCursor; - -/* -** When a sub-program is executed (OP_Program), a structure of this type -** is allocated to store the current value of the program counter, as -** well as the current memory cell array and various other frame specific -** values stored in the Vdbe struct. When the sub-program is finished, -** these values are copied back to the Vdbe from the VdbeFrame structure, -** restoring the state of the VM to as it was before the sub-program -** began executing. -** -** Frames are stored in a linked list headed at Vdbe.pParent. Vdbe.pParent -** is the parent of the current frame, or zero if the current frame -** is the main Vdbe program. -*/ -typedef struct VdbeFrame VdbeFrame; -struct VdbeFrame { - Vdbe *v; /* VM this frame belongs to */ - int pc; /* Program Counter */ - Op *aOp; /* Program instructions */ - int nOp; /* Size of aOp array */ - Mem *aMem; /* Array of memory cells */ - int nMem; /* Number of entries in aMem */ - VdbeCursor **apCsr; /* Element of Vdbe cursors */ - u16 nCursor; /* Number of entries in apCsr */ - void *token; /* Copy of SubProgram.token */ - int nChildMem; /* Number of memory cells for child frame */ - int nChildCsr; /* Number of cursors for child frame */ - i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ - int nChange; /* Statement changes (Vdbe.nChanges) */ - VdbeFrame *pParent; /* Parent of this frame */ -}; - -#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) - -/* -** A value for VdbeCursor.cacheValid that means the cache is always invalid. -*/ -#define CACHE_STALE 0 - -/* -** Internally, the vdbe manipulates nearly all SQL values as Mem -** structures. Each Mem struct may cache multiple representations (string, -** integer etc.) of the same value. A value (and therefore Mem structure) -** has the following properties: -** -** Each value has a manifest type. The manifest type of the value stored -** in a Mem struct is returned by the MemType(Mem*) macro. The type is -** one of SQLITE_NULL, SQLITE_INTEGER, SQLITE_REAL, SQLITE_TEXT or -** SQLITE_BLOB. -*/ -struct Mem { - union { - i64 i; /* Integer value. */ - int nZero; /* Used when bit MEM_Zero is set in flags */ - FuncDef *pDef; /* Used only when flags==MEM_Agg */ - RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ - VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ - } u; - double r; /* Real value */ - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ - int n; /* Number of characters in string value, excluding '\0' */ - u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ - u8 type; /* One of SQLITE_NULL, SQLITE_TEXT, SQLITE_INTEGER, etc */ - u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ -}; - -/* One or more of the following flags are set to indicate the validOK -** representations of the value stored in the Mem struct. -** -** If the MEM_Null flag is set, then the value is an SQL NULL value. -** No other flags may be set in this case. -** -** If the MEM_Str flag is set then Mem.z points at a string representation. -** Usually this is encoded in the same unicode encoding as the main -** database (see below for exceptions). If the MEM_Term flag is also -** set, then the string is nul terminated. The MEM_Int and MEM_Real -** flags may coexist with the MEM_Str flag. -** -** Multiple of these values can appear in Mem.flags. But only one -** at a time can appear in Mem.type. -*/ -#define MEM_Null 0x0001 /* Value is NULL */ -#define MEM_Str 0x0002 /* Value is a string */ -#define MEM_Int 0x0004 /* Value is an integer */ -#define MEM_Real 0x0008 /* Value is a real number */ -#define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_RowSet 0x0020 /* Value is a RowSet object */ -#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ -#define MEM_TypeMask 0x00ff /* Mask of type bits */ - -/* Whenever Mem contains a valid string or blob representation, one of -** the following flags must be set to determine the memory management -** policy for Mem.z. The MEM_Term flag tells us whether or not the -** string is \000 or \u0000 terminated -*/ -#define MEM_Term 0x0200 /* String rep is nul terminated */ -#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ -#define MEM_Static 0x0800 /* Mem.z points to a static string */ -#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ -#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ -#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ - -#ifdef SQLITE_OMIT_INCRBLOB - #undef MEM_Zero - #define MEM_Zero 0x0000 -#endif - - -/* -** Clear any existing type flags from a Mem and replace them with f -*/ -#define MemSetTypeFlag(p, f) \ - ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) - - -/* A VdbeFunc is just a FuncDef (defined in sqliteInt.h) that contains -** additional information about auxiliary information bound to arguments -** of the function. This is used to implement the sqlite3_get_auxdata() -** and sqlite3_set_auxdata() APIs. The "auxdata" is some auxiliary data -** that can be associated with a constant argument to a function. This -** allows functions such as "regexp" to compile their constant regular -** expression argument once and reused the compiled code for multiple -** invocations. -*/ -struct VdbeFunc { - FuncDef *pFunc; /* The definition of the function */ - int nAux; /* Number of entries allocated for apAux[] */ - struct AuxData { - void *pAux; /* Aux data for the i-th argument */ - void (*xDelete)(void *); /* Destructor for the aux data */ - } apAux[1]; /* One slot for each function argument */ -}; - -/* -** The "context" argument for a installable function. A pointer to an -** instance of this structure is the first argument to the routines used -** implement the SQL functions. -** -** There is a typedef for this structure in sqlite.h. So all routines, -** even the public interface to SQLite, can use a pointer to this structure. -** But this file is the only place where the internal details of this -** structure are known. -** -** This structure is defined inside of vdbeInt.h because it uses substructures -** (Mem) which are only defined there. -*/ -struct sqlite3_context { - FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ - VdbeFunc *pVdbeFunc; /* Auxilary data, if created. */ - Mem s; /* The return value is stored here */ - Mem *pMem; /* Memory cell used to store aggregate context */ - int isError; /* Error code returned by the function. */ - CollSeq *pColl; /* Collating sequence */ -}; - -/* -** A Set structure is used for quick testing to see if a value -** is part of a small set. Sets are used to implement code like -** this: -** x.y IN ('hi','hoo','hum') -*/ -typedef struct Set Set; -struct Set { - Hash hash; /* A set is just a hash table */ - HashElem *prev; /* Previously accessed hash elemen */ -}; - -/* -** An instance of the virtual machine. This structure contains the complete -** state of the virtual machine. -** -** The "sqlite3_stmt" structure pointer that is returned by sqlite3_compile() -** is really a pointer to an instance of this structure. -** -** The Vdbe.inVtabMethod variable is set to non-zero for the duration of -** any virtual table method invocations made by the vdbe program. It is -** set to 2 for xDestroy method calls and 1 for all other methods. This -** variable is used for two purposes: to allow xDestroy methods to execute -** "DROP TABLE" statements and to prevent some nasty side effects of -** malloc failure when SQLite is invoked recursively by a virtual table -** method function. -*/ -struct Vdbe { - sqlite3 *db; /* The database connection that owns this statement */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ - int nOp; /* Number of instructions in the program */ - int nOpAlloc; /* Number of slots allocated for aOp[] */ - Op *aOp; /* Space to hold the virtual machine's program */ - int nLabel; /* Number of labels used */ - int nLabelAlloc; /* Number of slots allocated in aLabel[] */ - int *aLabel; /* Space to hold the labels */ - Mem **apArg; /* Arguments to currently executing user function */ - Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ - u16 nResColumn; /* Number of columns in one row of the result set */ - u16 nCursor; /* Number of slots in apCsr[] */ - VdbeCursor **apCsr; /* One element of this array for each open cursor */ - u8 errorAction; /* Recovery action to do in case of an error */ - u8 okVar; /* True if azVar[] has been initialized */ - ynVar nVar; /* Number of entries in aVar[] */ - Mem *aVar; /* Values for the OP_Variable opcode. */ - char **azVar; /* Name of variables */ - u32 magic; /* Magic number for sanity checking */ - int nMem; /* Number of memory locations currently allocated */ - Mem *aMem; /* The memory locations */ - u32 cacheCtr; /* VdbeCursor row cache generation counter */ - int pc; /* The program counter */ - int rc; /* Value to return */ - char *zErrMsg; /* Error message written here */ - u8 explain; /* True if EXPLAIN present on SQL command */ - u8 changeCntOn; /* True to update the change-counter */ - u8 expired; /* True if the VM needs to be recompiled */ - u8 runOnlyOnce; /* Automatically expire on reset */ - u8 minWriteFileFormat; /* Minimum file format for writable database files */ - u8 inVtabMethod; /* See comments above */ - u8 usesStmtJournal; /* True if uses a statement journal */ - u8 readOnly; /* True for read-only statements */ - u8 isPrepareV2; /* True if prepared with prepare_v2() */ - int nChange; /* Number of db changes made since last reset */ - int btreeMask; /* Bitmask of db->aDb[] entries referenced */ - i64 startTime; /* Time when query started - used for profiling */ - BtreeMutexArray aMutex; /* An array of Btree used here and needing locks */ - int aCounter[3]; /* Counters used by sqlite3_stmt_status() */ - char *zSql; /* Text of the SQL statement that generated this */ - void *pFree; /* Free this when deleting the vdbe */ - i64 nFkConstraint; /* Number of imm. FK constraints this VM */ - i64 nStmtDefCons; /* Number of def. constraints when stmt started */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ -#ifdef SQLITE_DEBUG - FILE *trace; /* Write an execution trace here, if not NULL */ -#endif - VdbeFrame *pFrame; /* Parent frame */ - int nFrame; /* Number of frames in pFrame list */ - u32 expmask; /* Binding to these vars invalidates VM */ -}; - -/* -** The following are allowed values for Vdbe.magic -*/ -#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ - -/* -** Function prototypes -*/ -SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); -void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor*); -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); -#endif -SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int); -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, int, Mem*, int); -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); -SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(VdbeFunc*, int); - -int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); -SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); -SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); -SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); -SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); -SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); -#ifdef SQLITE_OMIT_FLOATING_POINT -# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 -#else -SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); -#endif -SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, int); -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); -SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); -SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); -SQLITE_PRIVATE void sqlite3VdbeMemReleaseExternal(Mem *p); -SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); -SQLITE_PRIVATE const char *sqlite3OpcodeName(int); -SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); -SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); -SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); -SQLITE_PRIVATE void sqlite3VdbeMemStoreType(Mem *pMem); - -#ifndef SQLITE_OMIT_FOREIGN_KEY -SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); -#else -# define sqlite3VdbeCheckFk(p,i) 0 -#endif - -#ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3VdbeMutexArrayEnter(Vdbe *p); -#else -# define sqlite3VdbeMutexArrayEnter(p) -#endif - -SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); -#endif -SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); - -#ifndef SQLITE_OMIT_INCRBLOB -SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); -#else - #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK -#endif - -#endif /* !defined(_VDBEINT_H_) */ - -/************** End of vdbeInt.h *********************************************/ -/************** Continuing where we left off in utf.c ************************/ #ifndef SQLITE_AMALGAMATION /* ** The following constant value is used by the SQLITE_BIGENDIAN and ** SQLITE_LITTLEENDIAN macros. @@ -19584,10 +20159,16 @@ #define UpperToLower sqlite3UpperToLower /* ** Some systems have stricmp(). Others have strcasecmp(). Because ** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-20522-24639 The sqlite3_strnicmp() API allows +** applications and extensions to compare the contents of two buffers +** containing UTF-8 strings in a case-independent fashion, using the same +** definition of case independence that SQLite uses internally when +** comparing identifiers. */ SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; @@ -19601,125 +20182,115 @@ while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } /* -** Return TRUE if z is a pure numeric string. Return FALSE and leave -** *realnum unchanged if the string contains any character which is not -** part of a number. -** -** If the string is pure numeric, set *realnum to TRUE if the string -** contains the '.' character or an "E+000" style exponentiation suffix. -** Otherwise set *realnum to FALSE. Note that just becaue *realnum is -** false does not mean that the number can be successfully converted into -** an integer - it might be too big. -** -** An empty string is considered non-numeric. -*/ -SQLITE_PRIVATE int sqlite3IsNumber(const char *z, int *realnum, u8 enc){ +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. +** +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. +** +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. Valid numbers +** are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. +*/ +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +#ifndef SQLITE_OMIT_FLOATING_POINT int incr = (enc==SQLITE_UTF8?1:2); - if( enc==SQLITE_UTF16BE ) z++; - if( *z=='-' || *z=='+' ) z += incr; - if( !sqlite3Isdigit(*z) ){ - return 0; - } - z += incr; - *realnum = 0; - while( sqlite3Isdigit(*z) ){ z += incr; } -#ifndef SQLITE_OMIT_FLOATING_POINT - if( *z=='.' ){ - z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } - if( *z=='e' || *z=='E' ){ - z += incr; - if( *z=='+' || *z=='-' ) z += incr; - if( !sqlite3Isdigit(*z) ) return 0; - while( sqlite3Isdigit(*z) ){ z += incr; } - *realnum = 1; - } -#endif - return *z==0; -} - -/* -** The string z[] is an ASCII representation of a real number. -** Convert this string to a double. -** -** This routine assumes that z[] really is a valid number. If it -** is not, the result is undefined. -** -** This routine is used instead of the library atof() function because -** the library atof() might want to use "," as the decimal point instead -** of "." depending on how locale is set. But that would cause problems -** for SQL. So this routine always uses "." regardless of locale. -*/ -SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult){ -#ifndef SQLITE_OMIT_FLOATING_POINT - const char *zBegin = z; + const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ - int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ - int d = 0; /* adjust exponent for shifting decimal point */ - int esign = 1; /* sign of exponent */ - int e = 0; /* exponent */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + *pResult = 0.0; /* Default return value, in case of an error */ + + if( enc==SQLITE_UTF16BE ) z++; + /* skip leading spaces */ - while( sqlite3Isspace(*z) ) z++; + while( z=zEnd ) return 0; + /* get sign of significand */ if( *z=='-' ){ sign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } + /* skip leading zeroes */ - while( z[0]=='0' ) z++, nDigits++; + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ - z++; + z+=incr; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ - while( sqlite3Isdigit(*z) && s<((LARGEST_INT64-9)/10) ){ + while( z=zEnd ) goto do_atof_calc; /* if exponent is present */ if( *z=='e' || *z=='E' ){ - z++; + z+=incr; + eValid = 0; + if( z>=zEnd ) goto do_atof_calc; /* get sign of exponent */ if( *z=='-' ){ esign = -1; - z++; + z+=incr; }else if( *z=='+' ){ - z++; + z+=incr; } /* copy digits to exponent */ - while( sqlite3Isdigit(*z) ){ + while( z=zEnd && nDigits>0 && eValid; #else - return sqlite3Atoi64(z, pResult); + return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } /* ** Compare the 19-character string zNum against the text representation ** value 2^63: 9223372036854775808. Return negative, zero, or positive ** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. ** ** Unlike memcmp() this routine is guaranteed to return the difference ** in the values of the last digit if the only difference is in the ** last digit. So, for example, ** -** compare2pow63("9223372036854775800") +** compare2pow63("9223372036854775800", 1) ** ** will return -8. */ -static int compare2pow63(const char *zNum){ - int c; - c = memcmp(zNum,"922337203685477580",18)*10; +static int compare2pow63(const char *zNum, int incr){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i*incr]-pow63[i])*10; + } if( c==0 ){ - c = zNum[18] - '8'; + c = zNum[18*incr] - '8'; testcase( c==(-1) ); testcase( c==0 ); testcase( c==(+1) ); } return c; } /* -** Return TRUE if zNum is a 64-bit signed integer and write -** the value of the integer into *pNum. If zNum is not an integer -** or is an integer that is too large to be expressed with 64 bits, -** then return false. +** Convert zNum to a 64-bit signed integer and write +** the value of the integer into *pNum. +** If zNum is exactly 9223372036854665808, return 2. +** This is a special case as the context will determine +** if it is too big (used as a negative). +** If zNum is not an integer or is an integer that +** is too large to be expressed with 64 bits, +** then return 1. Otherwise return 0. ** -** When this routine was originally written it dealt with only -** 32-bit numbers. At that time, it was much faster than the -** atoi() library routine in RedHat 7.2. +** length is the number of bytes in the string (bytes, not characters). +** The string is not necessarily zero-terminated. The encoding is +** given by enc. */ -SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum){ +SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ + int incr = (enc==SQLITE_UTF8?1:2); i64 v = 0; - int neg; - int i, c; + int neg = 0; /* assume positive */ + int i; + int c = 0; const char *zStart; - while( sqlite3Isspace(*zNum) ) zNum++; + const char *zEnd = zNum + length; + if( enc==SQLITE_UTF16BE ) zNum++; + while( zNum=zEnd ) goto do_atoi_calc; if( *zNum=='-' ){ neg = 1; - zNum++; + zNum+=incr; }else if( *zNum=='+' ){ - neg = 0; - zNum++; - }else{ - neg = 0; + zNum+=incr; } +do_atoi_calc: zStart = zNum; - while( zNum[0]=='0' ){ zNum++; } /* Skip over leading zeros. Ticket #2454 */ - for(i=0; (c=zNum[i])>='0' && c<='9'; i++){ + while( zNum='0' && c<='9'; i+=incr){ v = v*10 + c - '0'; } *pNum = neg ? -v : v; testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( c!=0 || (i==0 && zStart==zNum) || i>19 ){ + if( (c!=0 && &zNum[i]19*incr ){ /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranting that it is too large) */ - return 0; - }else if( i<19 ){ + ** than 19 digits (thus guaranteeing that it is too large) */ + return 1; + }else if( i<19*incr ){ /* Less than 19 digits, so we know that it fits in 64 bits */ - return 1; + return 0; }else{ /* 19-digit numbers must be no larger than 9223372036854775807 if positive ** or 9223372036854775808 if negative. Note that 9223372036854665808 - ** is 2^63. */ - return compare2pow63(zNum)='0' && zNum[0]<='9' ); /* zNum is an unsigned number */ - - if( negFlag ) neg = 1-neg; - while( *zNum=='0' ){ - zNum++; /* Skip leading zeros. Ticket #2454 */ - } - for(i=0; zNum[i]; i++){ assert( zNum[i]>='0' && zNum[i]<='9' ); } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( i<19 ){ - /* Guaranteed to fit if less than 19 digits */ - return 1; - }else if( i>19 ){ - /* Guaranteed to be too big if greater than 19 digits */ - return 0; - }else{ - /* Compare against 2^63. */ - return compare2pow63(zNum)0 ){ if( wrote<0 ){ /* lastErrno set by seekAndWrite */ return SQLITE_IOERR_WRITE; }else{ pFile->lastErrno = 0; /* not a system error */ return SQLITE_FULL; } } + return SQLITE_OK; } #ifdef SQLITE_TEST /* @@ -25400,16 +25956,27 @@ /* ** Truncate an open file to a specified size */ static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; int rc; - assert( id ); + assert( pFile ); SimulateIOError( return SQLITE_IOERR_TRUNCATE ); - rc = ftruncate(((unixFile*)id)->h, (off_t)nByte); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = ftruncate(pFile->h, (off_t)nByte); if( rc ){ - ((unixFile*)id)->lastErrno = errno; + pFile->lastErrno = errno; return SQLITE_IOERR_TRUNCATE; }else{ #ifndef NDEBUG /* If we are doing a normal write to a database file (as opposed to ** doing a hot-journal rollback or a write to some file other than a @@ -25416,12 +25983,12 @@ ** normal database file) and we truncate the file to zero length, ** that effectively updates the change counter. This might happen ** when restoring a database using the backup API from a zero-length ** source. */ - if( ((unixFile*)id)->inNormalWrite && nByte==0 ){ - ((unixFile*)id)->transCntrChng = 1; + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; } #endif return SQLITE_OK; } @@ -25460,10 +26027,58 @@ ** proxying locking division. */ static int proxyFileControl(sqlite3_file*,int,void*); #endif +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. +** +** If the user has configured a chunk-size for this file, it could be +** that the file needs to be extended at this point. Otherwise, the +** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( fstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( posix_fallocate(pFile->h, buf.st_size, nSize-buf.st_size) ){ + return SQLITE_IOERR_WRITE; + } +#else + /* If the OS does not have posix_fallocate(), fake it. First use + ** ftruncate() to set the file size, then write a single byte to + ** the last byte in each block within the extended region. This + ** is the same technique used by glibc to implement posix_fallocate() + ** on systems that do not have a real fallocate() system call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + i64 iWrite; /* Next offset to write to */ + int nWrite; /* Return value from seekAndWrite() */ + + if( ftruncate(pFile->h, nSize) ){ + pFile->lastErrno = errno; + return SQLITE_IOERR_TRUNCATE; + } + iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; + do { + nWrite = seekAndWrite(pFile, iWrite, "", 1); + iWrite += nBlk; + } while( nWrite==1 && iWritelastErrno; return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((unixFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { -#if 0 /* No performance advantage seen on Linux */ - sqlite3_int64 szFile = *(sqlite3_int64*)pArg; - unixFile *pFile = (unixFile*)id; - ftruncate(pFile->h, szFile); -#endif - return SQLITE_OK; + return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and ** it hence it is OK for the transaction change counter to be @@ -25702,74 +26316,100 @@ sqlite3_free(p); } } /* -** Open a shared-memory area associated with open database file fd. +** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. ** ** The file used to implement shared-memory is in the same directory ** as the open database file and has the same name as the open database ** file with the "-shm" suffix added. For example, if the database file ** is "/home/user1/config.db" then the file that is created and mmapped -** for shared memory will be called "/home/user1/config.db-shm". We -** experimented with using files in /dev/tmp or an some other tmpfs mount. -** But if a file in a different directory from the database file is used, -** then differing access permissions or a chroot() might cause two different -** processes on the same database to end up using different files for -** shared memory - meaning that their memory would not really be shared - -** resulting in database corruption. +** for shared memory will be called "/home/user1/config.db-shm". +** +** Another approach to is to use files in /dev/shm or /dev/tmp or an +** some other tmpfs mount. But if a file in a different directory +** from the database file is used, then differing access permissions +** or a chroot() might cause two different processes on the same +** database to end up using different files for shared memory - +** meaning that their memory would not really be shared - resulting +** in database corruption. Nevertheless, this tmpfs file usage +** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" +** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time +** option results in an incompatible build of SQLite; builds of SQLite +** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the +** same database file at the same time, database corruption will likely +** result. The SQLITE_SHM_DIRECTORY compile-time option is considered +** "unsupported" and may go away in a future SQLite release. ** ** When opening a new shared-memory file, if no other instances of that ** file are currently open, in this process or in other processes, then ** the file must be truncated to zero length or have its header cleared. */ -static int unixShmOpen( - sqlite3_file *fd /* The file descriptor of the associated database */ -){ - struct unixShm *p = 0; /* The connection to be opened */ - struct unixShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct unixFile *pDbFd; /* Underlying database file */ - unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ - int nShmFilename; /* Size of the SHM filename in bytes */ - - /* Allocate space for the new sqlite3_shm object. - */ +static int unixOpenSharedMemory(unixFile *pDbFd){ + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc; /* Result code */ + unixInodeInfo *pInode; /* The inode of fd */ + char *zShmFilename; /* Name of the file used for SHM */ + int nShmFilename; /* Size of the SHM filename in bytes */ + + /* Allocate space for the new unixShm object. */ p = sqlite3_malloc( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); - pDbFd = (struct unixFile*)fd; assert( pDbFd->pShm==0 ); - /* Check to see if a unixShmNode object already exists. Reuse an existing - ** one if present. Create a new one if necessary. + /* Check to see if a unixShmNode object already exists. Reuse an existing + ** one if present. Create a new one if necessary. */ unixEnterMutex(); pInode = pDbFd->pInode; pShmNode = pInode->pShmNode; if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ + + /* Call fstat() to figure out the permissions on the database file. If + ** a new *-shm file is created, an attempt will be made to create it + ** with the same permissions. The actual permissions the file is created + ** with are subject to the current umask setting. + */ + if( fstat(pDbFd->h, &sStat) ){ + rc = SQLITE_IOERR_FSTAT; + goto shm_open_err; + } + +#ifdef SQLITE_SHM_DIRECTORY + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 30; +#else nShmFilename = 5 + (int)strlen(pDbFd->zPath); +#endif pShmNode = sqlite3_malloc( sizeof(*pShmNode) + nShmFilename ); if( pShmNode==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)); zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; +#ifdef SQLITE_SHM_DIRECTORY + sqlite3_snprintf(nShmFilename, zShmFilename, + SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", + (u32)sStat.st_ino, (u32)sStat.st_dev); +#else sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", pDbFd->zPath); +#endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->mutex==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; } - pShmNode->h = open(zShmFilename, O_RDWR|O_CREAT, 0664); + pShmNode->h = open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); if( pShmNode->h<0 ){ rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; } @@ -25788,18 +26428,28 @@ if( rc ) goto shm_open_err; } /* Make the new connection a child of the unixShmNode */ p->pShmNode = pShmNode; - p->pNext = pShmNode->pFirst; #ifdef SQLITE_DEBUG p->id = pShmNode->nextShmId++; #endif - pShmNode->pFirst = p; pShmNode->nRef++; pDbFd->pShm = p; unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the unixEnterMutex() mutex and the pointer from the + ** new (struct unixShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); return SQLITE_OK; /* Jump here on any error */ shm_open_err: unixShmPurge(pDbFd); /* This call frees pShmNode if required */ @@ -25807,53 +26457,111 @@ unixLeaveMutex(); return rc; } /* -** Close a connection to shared-memory. Delete the underlying -** storage if deleteFlag is true. +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. */ -static int unixShmClose( - sqlite3_file *fd, /* The underlying database file */ - int deleteFlag /* Delete shared-memory if true */ +static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ ){ - unixShm *p; /* The connection to be closed */ - unixShmNode *pShmNode; /* The underlying shared-memory file */ - unixShm **pp; /* For looping over sibling connections */ - unixFile *pDbFd; /* The underlying database file */ - - pDbFd = (unixFile*)fd; - p = pDbFd->pShm; - if( p==0 ) return SQLITE_OK; + unixFile *pDbFd = (unixFile*)fd; + unixShm *p; + unixShmNode *pShmNode; + int rc = SQLITE_OK; + + /* If the shared-memory file has not yet been opened, open it now. */ + if( pDbFd->pShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + } + + p = pDbFd->pShm; pShmNode = p->pShmNode; - - assert( pShmNode==pDbFd->pInode->pShmNode ); - assert( pShmNode->pInode==pDbFd->pInode ); - - /* Remove connection p from the set of connections associated - ** with pShmNode */ sqlite3_mutex_enter(pShmNode->mutex); - for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} - *pp = p->pNext; + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); - /* Free the connection p */ - sqlite3_free(p); - pDbFd->pShm = 0; + if( pShmNode->nRegion<=iRegion ){ + char **apNew; /* New apRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + struct stat sStat; /* Used by fstat() */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + if( fstat(pShmNode->h, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + + if( sStat.st_sizeh, nByte) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, (iRegion+1)*sizeof(char *) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->apRegion = apNew; + while(pShmNode->nRegion<=iRegion){ + void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->h, pShmNode->nRegion*szRegion + ); + if( pMem==MAP_FAILED ){ + rc = SQLITE_IOERR; + goto shmpage_out; + } + pShmNode->apRegion[pShmNode->nRegion] = pMem; + pShmNode->nRegion++; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; + }else{ + *pp = 0; + } sqlite3_mutex_leave(pShmNode->mutex); - - /* If pShmNode->nRef has reached 0, then close the underlying - ** shared-memory file, too */ - unixEnterMutex(); - assert( pShmNode->nRef>0 ); - pShmNode->nRef--; - if( pShmNode->nRef==0 ){ - if( deleteFlag ) unlink(pShmNode->zFilename); - unixShmPurge(pDbFd); - } - unixLeaveMutex(); - - return SQLITE_OK; + return rc; } /* ** Change the lock state for a shared-memory segment. ** @@ -25979,111 +26687,64 @@ unixEnterMutex(); unixLeaveMutex(); } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion -** bytes in size. -** -** If an error occurs, an error code is returned and *pp is set to NULL. -** -** Otherwise, if the isWrite parameter is 0 and the requested shared-memory -** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** isWrite is non-zero and the requested shared-memory region has not yet -** been allocated, it is allocated by this function. -** -** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped -** memory and SQLITE_OK returned. -*/ -static int unixShmMap( - sqlite3_file *fd, /* Handle open on database file */ - int iRegion, /* Region to retrieve */ - int szRegion, /* Size of regions */ - int isWrite, /* True to extend file if necessary */ - void volatile **pp /* OUT: Mapped memory */ -){ - unixFile *pDbFd = (unixFile*)fd; - unixShm *p = pDbFd->pShm; - unixShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_OK; - +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +** +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. +*/ +static int unixShmUnmap( + sqlite3_file *fd, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ +){ + unixShm *p; /* The connection to be closed */ + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ + + pDbFd = (unixFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + + /* Remove connection p from the set of connections associated + ** with pShmNode */ sqlite3_mutex_enter(pShmNode->mutex); - assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); - - if( pShmNode->nRegion<=iRegion ){ - char **apNew; /* New apRegion[] array */ - int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ - struct stat sStat; /* Used by fstat() */ - - pShmNode->szRegion = szRegion; - - /* The requested region is not mapped into this processes address space. - ** Check to see if it has been allocated (i.e. if the wal-index file is - ** large enough to contain the requested region). - */ - if( fstat(pShmNode->h, &sStat) ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; - } - - if( sStat.st_sizeh, nByte) ){ - rc = SQLITE_IOERR_SHMSIZE; - goto shmpage_out; - } - } - - /* Map the requested memory region into this processes address space. */ - apNew = (char **)sqlite3_realloc( - pShmNode->apRegion, (iRegion+1)*sizeof(char *) - ); - if( !apNew ){ - rc = SQLITE_IOERR_NOMEM; - goto shmpage_out; - } - pShmNode->apRegion = apNew; - while(pShmNode->nRegion<=iRegion){ - void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, iRegion*szRegion - ); - if( pMem==MAP_FAILED ){ - rc = SQLITE_IOERR; - goto shmpage_out; - } - pShmNode->apRegion[pShmNode->nRegion] = pMem; - pShmNode->nRegion++; - } - } - -shmpage_out: - if( pShmNode->nRegion>iRegion ){ - *pp = pShmNode->apRegion[iRegion]; - }else{ - *pp = 0; - } + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; sqlite3_mutex_leave(pShmNode->mutex); - return rc; + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + unixEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + if( deleteFlag ) unlink(pShmNode->zFilename); + unixShmPurge(pDbFd); + } + unixLeaveMutex(); + + return SQLITE_OK; } + #else -# define unixShmOpen 0 +# define unixShmMap 0 # define unixShmLock 0 -# define unixShmMap 0 # define unixShmBarrier 0 -# define unixShmClose 0 +# define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -26137,15 +26798,14 @@ UNLOCK, /* xUnlock */ \ CKLOCK, /* xCheckReservedLock */ \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmOpen, /* xShmOpen */ \ + unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ - unixShmMap, /* xShmMap */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmClose /* xShmClose */ \ + unixShmUnmap /* xShmUnmap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ return &METHOD; \ } \ @@ -26158,29 +26818,29 @@ ** are also created. */ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* ShmOpen is enabled */ + 2, /* shared memory is enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ unixCheckReservedLock /* xCheckReservedLock method */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ nolockCheckReservedLock /* xCheckReservedLock method */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ dotlockIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ dotlockCheckReservedLock /* xCheckReservedLock method */ ) @@ -26187,11 +26847,11 @@ #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS IOMETHODS( flockIoFinder, /* Finder function name */ flockIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ flockCheckReservedLock /* xCheckReservedLock method */ ) @@ -26199,11 +26859,11 @@ #if OS_VXWORKS IOMETHODS( semIoFinder, /* Finder function name */ semIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ semClose, /* xClose method */ semLock, /* xLock method */ semUnlock, /* xUnlock method */ semCheckReservedLock /* xCheckReservedLock method */ ) @@ -26211,11 +26871,11 @@ #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE IOMETHODS( afpIoFinder, /* Finder function name */ afpIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ afpCheckReservedLock /* xCheckReservedLock method */ ) @@ -26236,11 +26896,11 @@ static int proxyUnlock(sqlite3_file*, int); static int proxyCheckReservedLock(sqlite3_file*, int*); IOMETHODS( proxyIoFinder, /* Finder function name */ proxyIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ proxyCheckReservedLock /* xCheckReservedLock method */ ) @@ -26249,11 +26909,11 @@ /* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE IOMETHODS( nfsIoFinder, /* Finder function name */ nfsIoMethods, /* sqlite3_io_methods object name */ - 1, /* ShmOpen is disabled */ + 1, /* shared memory is disabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ unixCheckReservedLock /* xCheckReservedLock method */ ) @@ -26396,16 +27056,26 @@ /* Parameter isDelete is only used on vxworks. Express this explicitly ** here to prevent compiler warnings about unused parameters. */ UNUSED_PARAMETER(isDelete); + + /* Usually the path zFilename should not be a relative pathname. The + ** exception is when opening the proxy "conch" file in builds that + ** include the special Apple locking styles. + */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + assert( zFilename==0 || zFilename[0]=='/' + || pVfs->pAppData==(void*)&autolockIoFinder ); +#else + assert( zFilename==0 || zFilename[0]=='/' ); +#endif OSTRACE(("OPEN %-3d %s\n", h, zFilename)); pNew->h = h; pNew->dirfd = dirfd; pNew->fileFlags = 0; - assert( zFilename==0 || zFilename[0]=='/' ); /* Never a relative pathname */ pNew->zPath = zFilename; #if OS_VXWORKS pNew->pId = vxworksFindFileId(zFilename); if( pNew->pId==0 ){ @@ -26711,10 +27381,70 @@ unixLeaveMutex(); } #endif /* if !OS_VXWORKS */ return pUnused; } + +/* +** This function is called by unixOpen() to determine the unix permissions +** to create new files with. If no error occurs, then SQLITE_OK is returned +** and a value suitable for passing as the third argument to open(2) is +** written to *pMode. If an IO error occurs, an SQLite error code is +** returned and the value of *pMode is not modified. +** +** If the file being opened is a temporary file, it is always created with +** the octal permissions 0600 (read/writable by owner only). If the file +** is a database or master journal file, it is created with the permissions +** mask SQLITE_DEFAULT_FILE_PERMISSIONS. +** +** Finally, if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions +** as the associated database file. +*/ +static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ + int flags, /* Flags passed as 4th argument to xOpen() */ + mode_t *pMode /* OUT: Permissions to open file with */ +){ + int rc = SQLITE_OK; /* Return Code */ + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + char zDb[MAX_PATHNAME+1]; /* Database file path */ + int nDb; /* Number of valid bytes in zDb */ + struct stat sStat; /* Output of stat() on database file */ + + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journal-NNNN" + ** "-wal-NNNN" + ** + ** where NNNN is a 4 digit decimal number. The NNNN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; + while( nDb>0 && zPath[nDb]!='l' ) nDb--; + nDb -= ((flags & SQLITE_OPEN_WAL) ? 3 : 7); + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + + if( 0==stat(zDb, &sStat) ){ + *pMode = sStat.st_mode & 0777; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + *pMode = 0600; + }else{ + *pMode = SQLITE_DEFAULT_FILE_PERMISSIONS; + } + return rc; +} /* ** Open the file zPath. ** ** Previously, the SQLite OS layer used three functions in place of this @@ -26762,13 +27492,15 @@ /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() ** is called the directory file descriptor will be fsync()ed and close()d. */ - int isOpenDirectory = (isCreate && - (eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL) - ); + int isOpenDirectory = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. */ char zTmpname[MAX_PATHNAME+1]; @@ -26784,21 +27516,22 @@ assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); assert(isCreate==0 || isReadWrite); assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, and master journal are never automatically - ** deleted. Nor are they ever temporary files. */ + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_TRANSIENT_DB + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ @@ -26832,11 +27565,17 @@ if( isCreate ) openFlags |= O_CREAT; if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); openFlags |= (O_LARGEFILE|O_BINARY); if( fd<0 ){ - mode_t openMode = (isDelete?0600:SQLITE_DEFAULT_FILE_PERMISSIONS); + mode_t openMode; /* Permissions to create file with */ + rc = findCreateFileMode(zName, flags, &openMode); + if( rc!=SQLITE_OK ){ + assert( !p->pUnused ); + assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); + return rc; + } fd = open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ /* Failed to open the file for read/write access. Try read-only. */ flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); @@ -26975,11 +27714,13 @@ int dirSync /* If true, fsync() directory after deleting file */ ){ int rc = SQLITE_OK; UNUSED_PARAMETER(NotUsed); SimulateIOError(return SQLITE_IOERR_DELETE); - unlink(zPath); + if( unlink(zPath)==(-1) && errno!=ENOENT ){ + return SQLITE_IOERR_DELETE; + } #ifndef SQLITE_DISABLE_DIRSYNC if( dirSync ){ int fd; rc = openDirectory(zPath, &fd); if( rc==SQLITE_OK ){ @@ -27102,11 +27843,11 @@ ** message is available, it is written to zBufOut. If no error message ** is available, zBufOut is left unmodified and SQLite uses a default ** error message. */ static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ - char *zErr; + const char *zErr; UNUSED_PARAMETER(NotUsed); unixEnterMutex(); zErr = dlerror(); if( zErr ){ sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); @@ -27239,11 +27980,11 @@ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; #if defined(NO_GETTOD) time_t t; time(&t); - *piNow = ((sqlite3_int64)i)*1000 + unixEpoch; + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; #elif OS_VXWORKS struct timespec sNow; clock_gettime(CLOCK_REALTIME, &sNow); *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; #else @@ -27646,17 +28387,20 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ struct timespec timeout = {1, 0}; /* 1 sec timeout */ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); +#if defined(__MAX_OS_X_VERSION_MIN_REQUIRED)\ + && __MAC_OS_X_VERSION_MIN_REQUIRED<1050 if( gethostuuid(pHostID, &timeout) ){ int err = errno; if( pError ){ *pError = err; } return SQLITE_IOERR; } +#endif #ifdef SQLITE_TEST /* simulate multiple hosts by creating unique hostid file paths */ if( sqlite3_hostid_num != 0){ pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); } @@ -27693,31 +28437,31 @@ /* create a new path by replace the trailing '-conch' with '-break' */ pathLen = strlcpy(tPath, cPath, MAXPATHLEN); if( pathLen>MAXPATHLEN || pathLen<6 || (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ - sprintf(errmsg, "path error (len %d)", (int)pathLen); + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); goto end_breaklock; } /* read the conch content */ readLen = pread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); if( readLenh); @@ -28243,11 +28987,11 @@ }else{ if( pCtx->conchFile ){ pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } OSTRACE(("TRANSPROXY %d %s\n", pFile->h, (rc==SQLITE_OK ? "ok" : "failed"))); @@ -28434,13 +29178,13 @@ } rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); if( rc ) return rc; sqlite3_free(conchFile); } - sqlite3_free(pCtx->lockProxyPath); + sqlite3DbFree(0, pCtx->lockProxyPath); sqlite3_free(pCtx->conchFilePath); - sqlite3_free(pCtx->dbPath); + sqlite3DbFree(0, pCtx->dbPath); /* restore the original locking context and pMethod then close it */ pFile->lockingContext = pCtx->oldLockingContext; pFile->pMethod = pCtx->pOldMethod; sqlite3_free(pCtx); return pFile->pMethod->xClose(id); @@ -28882,10 +29626,11 @@ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ DWORD lastErrno; /* The Windows errno from the last I/O error */ DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE WCHAR *zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ HANDLE hShared; /* Shared memory segment used for locking */ winceLock local; /* Locks obtained by this instance of winFile */ @@ -29392,10 +30137,46 @@ /***************************************************************************** ** The next group of routines implement the I/O methods specified ** by the sqlite3_io_methods object. ******************************************************************************/ + +/* +** Some microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occured, it is also necessary to call + ** GetLastError(). + */ + dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + if( (dwRet==INVALID_SET_FILE_POINTER && GetLastError()!=NO_ERROR) ){ + pFile->lastErrno = GetLastError(); + return 1; + } + + return 0; +} /* ** Close a file. ** ** It is reported that an attempt to close a handle might sometimes @@ -29435,17 +30216,10 @@ OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); OpenCounter(-1); return rc ? SQLITE_OK : SQLITE_IOERR; } -/* -** Some microsoft compilers lack this definition. -*/ -#ifndef INVALID_SET_FILE_POINTER -# define INVALID_SET_FILE_POINTER ((DWORD)-1) -#endif - /* ** Read data from a file into a buffer. Return SQLITE_OK if all ** bytes were read successfully and SQLITE_IOERR if anything goes ** wrong. */ @@ -29453,112 +30227,108 @@ sqlite3_file *id, /* File to read from */ void *pBuf, /* Write content into this buffer */ int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD got; + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &got, 0) ){ + if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ pFile->lastErrno = GetLastError(); return SQLITE_IOERR_READ; } - if( got==(DWORD)amt ){ - return SQLITE_OK; - }else{ + if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ - memset(&((char*)pBuf)[got], 0, amt-got); + memset(&((char*)pBuf)[nRead], 0, amt-nRead); return SQLITE_IOERR_SHORT_READ; } + + return SQLITE_OK; } /* ** Write data from a buffer into a file. Return SQLITE_OK on success ** or some other error code on failure. */ static int winWrite( - sqlite3_file *id, /* File to write into */ - const void *pBuf, /* The bytes to be written */ - int amt, /* Number of bytes to write */ - sqlite3_int64 offset /* Offset into the file to begin writing at */ + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - LONG upperBits = (LONG)((offset>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(offset & 0xffffffff); - DWORD rc; - winFile *pFile = (winFile*)id; - DWORD error; - DWORD wrote = 0; - - assert( id!=0 ); + int rc; /* True if error has occured, else false */ + winFile *pFile = (winFile*)id; /* File handle */ + + assert( amt>0 ); + assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); - rc = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( rc==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; - } - } - assert( amt>0 ); - while( - amt>0 - && (rc = WriteFile(pFile->h, pBuf, amt, &wrote, 0))!=0 - && wrote>0 - ){ - amt -= wrote; - pBuf = &((char*)pBuf)[wrote]; - } - if( !rc || amt>(int)wrote ){ - pFile->lastErrno = GetLastError(); - if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ - return SQLITE_FULL; - }else{ - return SQLITE_IOERR_WRITE; - } + + OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + + rc = seekWinFile(pFile, offset); + if( rc==0 ){ + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + + while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = GetLastError(); + rc = 1; + } + } + + if( rc ){ + if( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ){ + return SQLITE_FULL; + } + return SQLITE_IOERR_WRITE; } return SQLITE_OK; } /* ** Truncate an open file to a specified size */ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ - LONG upperBits = (LONG)((nByte>>32) & 0x7fffffff); - LONG lowerBits = (LONG)(nByte & 0xffffffff); - DWORD dwRet; - winFile *pFile = (winFile*)id; - DWORD error; - int rc = SQLITE_OK; - - assert( id!=0 ); + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + + assert( pFile ); + OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - dwRet = SetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); - if( dwRet==INVALID_SET_FILE_POINTER && (error=GetLastError())!=NO_ERROR ){ - pFile->lastErrno = error; + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( seekWinFile(pFile, nByte) ){ rc = SQLITE_IOERR_TRUNCATE; - /* SetEndOfFile will fail if nByte is negative */ - }else if( !SetEndOfFile(pFile->h) ){ + }else if( 0==SetEndOfFile(pFile->h) ){ pFile->lastErrno = GetLastError(); rc = SQLITE_IOERR_TRUNCATE; } - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc==SQLITE_OK ? "ok" : "failed")); + + OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; } #ifdef SQLITE_TEST /* @@ -29918,10 +30688,14 @@ return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)((winFile*)id)->lastErrno; return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + ((winFile*)id)->szChunk = *(int *)pArg; + return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { sqlite3_int64 sz = *(sqlite3_int64*)pArg; SimulateIOErrorBenign(1); winTruncate(id, sz); @@ -29953,17 +30727,20 @@ static int winDeviceCharacteristics(sqlite3_file *id){ UNUSED_PARAMETER(id); return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; } -/**************************************************************************** -********************************* Shared Memory ***************************** -** -** The next subdivision of code manages the shared-memory primitives. -*/ #ifndef SQLITE_OMIT_WAL +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +SYSTEM_INFO winSysInfo; + /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the winLockInfo objects used by ** this file, all of which may be shared by multiple threads. ** @@ -29971,11 +30748,11 @@ ** is held when required. This function is only used as part of assert() ** statements. e.g. ** ** winShmEnterMutex() ** assert( winShmMutexHeld() ); -** winEnterLeave() +** winShmLeaveMutex() */ static void winShmEnterMutex(void){ sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } static void winShmLeaveMutex(void){ @@ -30007,15 +30784,10 @@ ** ** Either winShmNode.mutex must be held or winShmNode.nRef==0 and ** winShmMutexHeld() is true when reading or writing any other field ** in this structure. ** -** To avoid deadlocks, mutex and mutexBuf are always released in the -** reverse order that they are acquired. mutexBuf is always acquired -** first and released last. This invariant is check by asserting -** sqlite3_mutex_notheld() on mutex whenever mutexBuf is acquired or -** released. */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ char *zFilename; /* Name of the file */ winFile hFile; /* File handle from winOpen */ @@ -30045,21 +30817,25 @@ /* ** Structure used internally by this VFS to record the state of an ** open shared memory connection. ** -** winShm.pFile->mutex must be held while reading or writing the -** winShm.pNext and winShm.locks[] elements. +** The following fields are initialized when this object is created and +** are read-only thereafter: ** -** The winShm.pFile element is initialized when the object is created -** and is read-only thereafter. +** winShm.pShmNode +** winShm.id +** +** All other fields are read/write. The winShm.pShmNode->mutex must be held +** while accessing any read/write fields. */ struct winShm { winShmNode *pShmNode; /* The underlying winShmNode object */ winShm *pNext; /* Next winShm with the same winShmNode */ u8 hasMutex; /* True if holding the winShmNode mutex */ - u8 hasMutexBuf; /* True if holding pFile->mutexBuf */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ #ifdef SQLITE_DEBUG u8 id; /* Id of this connection with its winShmNode */ #endif }; @@ -30090,27 +30866,32 @@ /* Initialize the locking parameters */ dwFlags = LOCKFILE_FAIL_IMMEDIATELY; if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; - /* Find the first bit in lockMask that is set */ memset(&ovlp, 0, sizeof(OVERLAPPED)); ovlp.Offset = ofst; /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = UnlockFileEx(pFile->hFile.h, 0, nByte, 0, &ovlp); }else{ rc = LockFileEx(pFile->hFile.h, dwFlags, 0, nByte, 0, &ovlp); } - if( !rc ){ - OSTRACE(("SHM-LOCK %d %s ERROR 0x%08lx\n", - pFile->hFile.h, - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - GetLastError())); + + if( rc!= 0 ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = GetLastError(); + rc = SQLITE_BUSY; } - rc = (rc!=0) ? SQLITE_OK : SQLITE_BUSY; + + OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", + pFile->hFile.h, + rc==SQLITE_OK ? "ok" : "failed", + lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", + pFile->lastErrno)); return rc; } /* Forward references to VFS methods */ @@ -30124,19 +30905,26 @@ ** by VFS shared-memory methods. */ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode **pp; winShmNode *p; + BOOL bRc; assert( winShmMutexHeld() ); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ int i; if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ - UnmapViewOfFile(p->aRegion[i].pMap); - CloseHandle(p->aRegion[i].hMap); + bRc = UnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); + bRc = CloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", + (int)GetCurrentProcessId(), i, + bRc ? "ok" : "failed")); } if( p->hFile.h != INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); winClose((sqlite3_file *)&p->hFile); SimulateIOErrorBenign(0); @@ -30154,35 +30942,23 @@ } } } /* -** Open a shared-memory area. This particular implementation uses -** mmapped files. -** -** zName is a filename used to identify the shared-memory area. The -** implementation does not (and perhaps should not) use this name -** directly, but rather use it as a template for finding an appropriate -** name for the shared-memory storage. In this implementation, the -** string "-index" is appended to zName and used as the name of the -** mmapped file. +** Open the shared-memory area associated with database file pDbFd. ** ** When opening a new shared-memory file, if no other instances of that ** file are currently open, in this process or in other processes, then ** the file must be truncated to zero length or have its header cleared. */ -static int winShmOpen( - sqlite3_file *fd /* The file to which to attach shared memory */ -){ - struct winFile *pDbFd; /* Database to which to attach SHM */ +static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ int rc; /* Result code */ struct winShmNode *pNew; /* Newly allocated winShmNode */ int nName; /* Size of zName in bytes */ - pDbFd = (winFile*)fd; assert( pDbFd->pShm==0 ); /* Not previously opened */ /* Allocate space for the new sqlite3_shm object. Also speculatively ** allocate space for a new winShmNode and filename. */ @@ -30221,14 +30997,15 @@ pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); if( pShmNode->mutex==0 ){ rc = SQLITE_NOMEM; goto shm_open_err; } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ + SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, /* Mode flags */ 0); if( SQLITE_OK!=rc ){ rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; } @@ -30249,18 +31026,28 @@ if( rc ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ p->pShmNode = pShmNode; - p->pNext = pShmNode->pFirst; #ifdef SQLITE_DEBUG p->id = pShmNode->nextShmId++; #endif - pShmNode->pFirst = p; pShmNode->nRef++; pDbFd->pShm = p; winShmLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the winShmEnterMutex() mutex and the pointer from the + ** new (struct winShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); return SQLITE_OK; /* Jump here on any error */ shm_open_err: winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); @@ -30273,11 +31060,11 @@ /* ** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. */ -static int winShmClose( +static int winShmUnmap( sqlite3_file *fd, /* Database holding shared memory */ int deleteFlag /* Delete after closing if true */ ){ winFile *pDbFd; /* Database holding shared-memory */ winShm *p; /* The connection to be closed */ @@ -30284,10 +31071,11 @@ winShmNode *pShmNode; /* The underlying shared-memory file */ winShm **pp; /* For looping over sibling connections */ pDbFd = (winFile*)fd; p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; pShmNode = p->pShmNode; /* Remove connection p from the set of connections associated ** with pShmNode */ sqlite3_mutex_enter(pShmNode->mutex); @@ -30309,10 +31097,131 @@ } winShmLeaveMutex(); return SQLITE_OK; } + +/* +** Change the lock state for a shared-memory segment. +*/ +static int winShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ + winShm *p = pDbFd->pShm; /* The shared memory being locked */ + winShm *pX; /* For looping over all siblings */ + winShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + + mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", + p->id, (int)GetCurrentProcessId(), p->sharedMask, p->exclMask, + rc ? "failed" : "ok")); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void winShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + /* MemoryBarrier(); // does not work -- do not know why not */ + winShmEnterMutex(); + winShmLeaveMutex(); +} /* ** This function is called to obtain a pointer to region iRegion of the ** shared-memory associated with the database file fd. Shared-memory regions ** are numbered starting from zero. Each shared-memory region is szRegion @@ -30338,13 +31247,20 @@ int isWrite, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ winFile *pDbFd = (winFile*)fd; winShm *p = pDbFd->pShm; - winShmNode *pShmNode = p->pShmNode; + winShmNode *pShmNode; int rc = SQLITE_OK; + if( !p ){ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + p = pDbFd->pShm; + } + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->mutex); assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ struct ShmRegion *apNew; /* New aRegion[] array */ @@ -30357,10 +31273,11 @@ ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); if( rc!=SQLITE_OK ){ + rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; } if( szhFile, nByte); if( rc!=SQLITE_OK ){ + rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; } } /* Map the requested memory region into this processes address space. */ @@ -30391,14 +31309,22 @@ void *pMap = 0; /* Mapped memory region */ hMap = CreateFileMapping(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); + OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, - 0, 0, nByte + 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); + OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", + (int)GetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, + pMap ? "ok" : "failed")); } if( !pMap ){ pShmNode->lastErrno = GetLastError(); rc = SQLITE_IOERR; if( hMap ) CloseHandle(hMap); @@ -30411,111 +31337,64 @@ } } shmpage_out: if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; char *p = (char *)pShmNode->aRegion[iRegion].pMap; - *pp = (void *)&p[iRegion*szRegion]; + *pp = (void *)&p[iOffsetShift]; }else{ *pp = 0; } sqlite3_mutex_leave(pShmNode->mutex); return rc; } -/* -** Change the lock state for a shared-memory segment. -*/ -static int winShmLock( - sqlite3_file *fd, /* Database file holding the shared memory */ - int ofst, /* First lock to acquire or release */ - int n, /* Number of locks to acquire or release */ - int flags /* What to do with the lock */ -){ - winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; - winShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_PROTOCOL; - - assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); - assert( n>=1 ); - assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) - || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); - assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - - sqlite3_mutex_enter(pShmNode->mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); - }else if( flags & SQLITE_SHM_SHARED ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); - }else{ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); - } - sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d %s\n", - p->id, (int)GetCurrentProcessId(), rc ? "failed" : "ok")); - return rc; -} - -/* -** Implement a memory barrier or memory fence on shared memory. -** -** All loads and stores begun before the barrier must complete before -** any load or store begun after the barrier. -*/ -static void winShmBarrier( - sqlite3_file *fd /* Database holding the shared memory */ -){ - UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); - winShmLeaveMutex(); -} - #else -# define winShmOpen 0 # define winShmMap 0 # define winShmLock 0 # define winShmBarrier 0 -# define winShmClose 0 +# define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ + /* -***************************** End Shared Memory ***************************** -****************************************************************************/ +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ /* ** This vector defines all the methods that can operate on an ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ - winClose, - winRead, - winWrite, - winTruncate, - winSync, - winFileSize, - winLock, - winUnlock, - winCheckReservedLock, - winFileControl, - winSectorSize, - winDeviceCharacteristics, - winShmOpen, /* xShmOpen */ - winShmLock, /* xShmLock */ - winShmMap, /* xShmMap */ - winShmBarrier, /* xShmBarrier */ - winShmClose /* xShmClose */ + 2, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winLock, /* xLock */ + winUnlock, /* xUnlock */ + winCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap /* xShmUnmap */ }; -/*************************************************************************** -** Here ends the I/O methods that form the sqlite3_io_methods object. +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** ** -** The next block of code implements the VFS methods. -****************************************************************************/ +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ /* ** Convert a UTF-8 filename into whatever form the underlying ** operating system wants filenames in. Space to hold the result ** is obtained from malloc and must be freed by the calling @@ -30688,13 +31567,64 @@ DWORD dwFlagsAndAttributes = 0; #if SQLITE_OS_WINCE int isTemp = 0; #endif winFile *pFile = (winFile*)id; - void *zConverted; /* Filename in OS encoding */ - const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ - char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATH+1]; /* Buffer used to create temp filename */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); +#ifndef NDEBUG + int isReadonly = (flags & SQLITE_OPEN_READONLY); +#endif + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and master journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); assert( id!=0 ); UNUSED_PARAMETER(pVfs); pFile->h = INVALID_HANDLE_VALUE; @@ -30701,11 +31631,12 @@ /* If the second argument to this function is NULL, generate a ** temporary file name to use */ if( !zUtf8Name ){ - int rc = getTempname(MAX_PATH+1, zTmpname); + assert(isDelete && !isOpenJournal); + rc = getTempname(MAX_PATH+1, zTmpname); if( rc!=SQLITE_OK ){ return rc; } zUtf8Name = zTmpname; } @@ -30714,33 +31645,35 @@ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ return SQLITE_NOMEM; } - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; }else{ dwDesiredAccess = GENERIC_READ; } + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is ** created. SQLite doesn't use it to indicate "exclusive access" ** as it is usually understood. */ - assert(!(flags & SQLITE_OPEN_EXCLUSIVE) || (flags & SQLITE_OPEN_CREATE)); - if( flags & SQLITE_OPEN_EXCLUSIVE ){ + if( isExclusive ){ /* Creates a new file, only if it does not already exist. */ /* If the file exists, it fails. */ dwCreationDisposition = CREATE_NEW; - }else if( flags & SQLITE_OPEN_CREATE ){ + }else if( isCreate ){ /* Open existing file, or create if it doesn't exist */ dwCreationDisposition = OPEN_ALWAYS; }else{ /* Opens a file, only if it exists. */ dwCreationDisposition = OPEN_EXISTING; } + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; - if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + + if( isDelete ){ #if SQLITE_OS_WINCE dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; isTemp = 1; #else dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY @@ -30753,10 +31686,11 @@ /* Reports from the internet are that performance is always ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ #if SQLITE_OS_WINCE dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; #endif + if( isNT() ){ h = CreateFileW((WCHAR*)zConverted, dwDesiredAccess, dwShareMode, NULL, @@ -30778,40 +31712,45 @@ dwFlagsAndAttributes, NULL ); #endif } + OSTRACE(("OPEN %d %s 0x%lx %s\n", h, zName, dwDesiredAccess, h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + if( h==INVALID_HANDLE_VALUE ){ + pFile->lastErrno = GetLastError(); free(zConverted); - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ return winOpen(pVfs, zName, id, - ((flags|SQLITE_OPEN_READONLY)&~SQLITE_OPEN_READWRITE), pOutFlags); + ((flags|SQLITE_OPEN_READONLY)&~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); }else{ return SQLITE_CANTOPEN_BKPT; } } + if( pOutFlags ){ - if( flags & SQLITE_OPEN_READWRITE ){ + if( isReadWrite ){ *pOutFlags = SQLITE_OPEN_READWRITE; }else{ *pOutFlags = SQLITE_OPEN_READONLY; } } + memset(pFile, 0, sizeof(*pFile)); pFile->pMethod = &winIoMethod; pFile->h = h; pFile->lastErrno = NO_ERROR; pFile->pVfs = pVfs; pFile->pShm = 0; pFile->zPath = zName; pFile->sectorSize = getSectorSize(pVfs, zUtf8Name); + #if SQLITE_OS_WINCE - if( (flags & (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB)) == - (SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB) + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && !winceCreateLock(zName, pFile) ){ CloseHandle(h); free(zConverted); return SQLITE_CANTOPEN_BKPT; @@ -30821,12 +31760,13 @@ }else #endif { free(zConverted); } + OpenCounter(+1); - return SQLITE_OK; + return rc; } /* ** Delete the named file. ** @@ -31340,10 +32280,17 @@ winSleep, /* xSleep */ winCurrentTime, /* xCurrentTime */ winGetLastError, /* xGetLastError */ winCurrentTimeInt64, /* xCurrentTimeInt64 */ }; + +#ifndef SQLITE_OMIT_WAL + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); + GetSystemInfo(&winSysInfo); + assert(winSysInfo.dwAllocationGranularity > 0); +#endif sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; } SQLITE_API int sqlite3_os_end(void){ @@ -31390,11 +32337,11 @@ ** start of a transaction, and is thus usually less than a few thousand, ** but can be as large as 2 billion for a really big database. */ /* Size of the Bitvec structure in bytes. */ -#define BITVEC_SZ (sizeof(void*)*128) /* 512 on 32bit. 1024 on 64bit */ +#define BITVEC_SZ 512 /* Round the union size down to the nearest pointer boundary, since that's how ** it will be aligned within the Bitvec struct. */ #define BITVEC_USIZE (((BITVEC_SZ-(3*sizeof(u32)))/sizeof(Bitvec*))*sizeof(Bitvec*)) @@ -31905,16 +32852,20 @@ ** Initialize and shutdown the page cache subsystem. Neither of these ** functions are threadsafe. */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ if( sqlite3GlobalConfig.pcache.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ sqlite3PCacheSetDefault(); } return sqlite3GlobalConfig.pcache.xInit(sqlite3GlobalConfig.pcache.pArg); } SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ if( sqlite3GlobalConfig.pcache.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ sqlite3GlobalConfig.pcache.xShutdown(sqlite3GlobalConfig.pcache.pArg); } } /* @@ -32371,12 +33322,17 @@ typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; -/* Pointers to structures of this type are cast and returned as -** opaque sqlite3_pcache* handles +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable ** flag (bPurgeable) are set when the cache is created. nMax may be ** modified at any time by a call to the pcache1CacheSize() method. @@ -32432,10 +33388,13 @@ int nCurrentPage; /* Number of purgeable pages allocated */ PgHdr1 *pLruHead, *pLruTail; /* LRU list of unpinned pages */ /* Variables related to SQLITE_CONFIG_PAGECACHE settings. */ int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nFreeSlot; /* Number of unused pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ void *pStart, *pEnd; /* Bounds of pagecache malloc range */ PgFreeslot *pFree; /* Free page blocks */ int isInit; /* True if initialized */ } pcache1_g; @@ -32479,10 +33438,12 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; pcache1.pFree = 0; while( n-- ){ p = (PgFreeslot*)pBuf; p->pNext = pcache1.pFree; @@ -32500,15 +33461,17 @@ ** back to sqlite3Malloc(). */ static void *pcache1Alloc(int nByte){ void *p; assert( sqlite3_mutex_held(pcache1.mutex) ); + sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); if( nByte<=pcache1.szSlot && pcache1.pFree ){ assert( pcache1.isInit ); p = (PgHdr1 *)pcache1.pFree; pcache1.pFree = pcache1.pFree->pNext; - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + pcache1.nFreeSlot--; + assert( pcache1.nFreeSlot>=0 ); sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1); }else{ /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the ** global pcache mutex and unlock the pager-cache object pCache. This is @@ -32538,10 +33501,12 @@ PgFreeslot *pSlot; sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, -1); pSlot = (PgFreeslot*)p; pSlot->pNext = pcache1.pFree; pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + assert( pcache1.nFreeSlot<=pcache1.nSlot ); }else{ int iSize; assert( sqlite3MemdebugHasType(p, MEMTYPE_PCACHE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); iSize = sqlite3MallocSize(p); @@ -32548,10 +33513,29 @@ sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, -iSize); sqlite3_free(p); } } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** Return the size of a pcache allocation +*/ +static int pcache1MemSize(void *p){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( p>=pcache1.pStart && pszPage; @@ -32604,10 +33588,36 @@ pcache1EnterMutex(); pcache1Free(p); pcache1LeaveMutex(); } + +/* +** Return true if it desirable to avoid allocating a new page cache +** entry. +** +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory put the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. +*/ +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + assert( sqlite3_mutex_held(pcache1.mutex) ); + if( pcache1.nSlot && pCache->szPage<=pcache1.szSlot ){ + return pcache1.nFreeSlotnPage - pCache->nRecyclable; if( createFlag==1 && ( nPinned>=(pcache1.nMaxPage+pCache->nMin-pcache1.nMinPage) || nPinned>=(pCache->nMax * 9 / 10) + || pcache1UnderMemoryPressure(pCache) )){ goto fetch_out; } if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ @@ -32905,11 +33921,13 @@ goto fetch_out; } /* Step 4. Try to recycle a page buffer if appropriate. */ if( pCache->bPurgeable && pcache1.pLruTail && ( - (pCache->nPage+1>=pCache->nMax) || pcache1.nCurrentPage>=pcache1.nMaxPage + (pCache->nPage+1>=pCache->nMax) + || pcache1.nCurrentPage>=pcache1.nMaxPage + || pcache1UnderMemoryPressure(pCache) )){ pPage = pcache1.pLruTail; pcache1RemoveFromHash(pPage); pcache1PinPage(pPage); if( pPage->pCache->szPage!=pCache->szPage ){ @@ -33048,10 +34066,11 @@ ** ** Destroy a cache allocated using pcache1Create(). */ static void pcache1Destroy(sqlite3_pcache *p){ PCache1 *pCache = (PCache1 *)p; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(); pcache1TruncateUnsafe(pCache, 0); pcache1.nMaxPage -= pCache->nMax; pcache1.nMinPage -= pCache->nMin; pcache1EnforceMaxPage(); @@ -33095,12 +34114,12 @@ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; if( pcache1.pStart==0 ){ PgHdr1 *p; pcache1EnterMutex(); - while( (nReq<0 || nFreefd)) #define FILEHANDLEID(fd) ((int)fd) /* -** The page cache as a whole is always in one of the following -** states: -** -** PAGER_UNLOCK The page cache is not currently reading or -** writing the database file. There is no -** data held in memory. This is the initial -** state. -** -** PAGER_SHARED The page cache is reading the database. -** Writing is not permitted. There can be -** multiple readers accessing the same database -** file at the same time. -** -** PAGER_RESERVED This process has reserved the database for writing -** but has not yet made any changes. Only one process -** at a time can reserve the database. The original -** database file has not been modified so other -** processes may still be reading the on-disk -** database file. -** -** PAGER_EXCLUSIVE The page cache is writing the database. -** Access is exclusive. No other processes or -** threads can be reading or writing while one -** process is writing. -** -** PAGER_SYNCED The pager moves to this state from PAGER_EXCLUSIVE -** after all dirty pages have been written to the -** database file and the file has been synced to -** disk. All that remains to do is to remove or -** truncate the journal file and the transaction -** will be committed. -** -** The page cache comes up in PAGER_UNLOCK. The first time a -** sqlite3PagerGet() occurs, the state transitions to PAGER_SHARED. -** After all pages have been released using sqlite_page_unref(), -** the state transitions back to PAGER_UNLOCK. The first time -** that sqlite3PagerWrite() is called, the state transitions to -** PAGER_RESERVED. (Note that sqlite3PagerWrite() can only be -** called on an outstanding page which means that the pager must -** be in PAGER_SHARED before it transitions to PAGER_RESERVED.) -** PAGER_RESERVED means that there is an open rollback journal. -** The transition to PAGER_EXCLUSIVE occurs before any changes -** are made to the database file, though writes to the rollback -** journal occurs with just PAGER_RESERVED. After an sqlite3PagerRollback() -** or sqlite3PagerCommitPhaseTwo(), the state can go back to PAGER_SHARED, -** or it can stay at PAGER_EXCLUSIVE if we are in exclusive access mode. -*/ -#define PAGER_UNLOCK 0 -#define PAGER_SHARED 1 /* same as SHARED_LOCK */ -#define PAGER_RESERVED 2 /* same as RESERVED_LOCK */ -#define PAGER_EXCLUSIVE 4 /* same as EXCLUSIVE_LOCK */ -#define PAGER_SYNCED 5 +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. +** +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ +** +** +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] +** +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] +** +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** +** +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occured, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). +*/ +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) /* ** A macro used for invoking the codec if there is one */ #ifdef SQLITE_HAS_CODEC @@ -33894,37 +35146,32 @@ u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ #endif }; /* -** A open page cache is an instance of the following structure. -** -** errCode -** -** Pager.errCode may be set to SQLITE_IOERR, SQLITE_CORRUPT, or -** or SQLITE_FULL. Once one of the first three errors occurs, it persists -** and is returned as the result of every major pager API call. The -** SQLITE_FULL return code is slightly different. It persists only until the -** next successful rollback is performed on the pager cache. Also, -** SQLITE_FULL does not affect the sqlite3PagerGet() and sqlite3PagerLookup() -** APIs, they may still be used successfully. -** -** dbSizeValid, dbSize, dbOrigSize, dbFileSize -** -** Managing the size of the database file in pages is a little complicated. -** The variable Pager.dbSize contains the number of pages that the database -** image currently contains. As the database image grows or shrinks this -** variable is updated. The variable Pager.dbFileSize contains the number -** of pages in the database file. This may be different from Pager.dbSize -** if some pages have been appended to the database image but not yet written -** out from the cache to the actual file on disk. Or if the image has been -** truncated by an incremental-vacuum operation. The Pager.dbOrigSize variable -** contains the number of pages in the database image when the current -** transaction was opened. The contents of all three of these variables is -** only guaranteed to be correct if the boolean Pager.dbSizeValid is true. -** -** TODO: Under what conditions is dbSizeValid set? Cleared? +** A open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: +** +** eState +** +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. +** +** eLock +** +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. ** ** changeCountDone ** ** This boolean variable is used to make sure that the change-counter ** (the 4-byte header field at byte offset 24 of the database file) is @@ -33939,104 +35186,157 @@ ** ** This mechanism means that when running in exclusive mode, a connection ** need only update the change-counter once, for the first transaction ** committed. ** -** dbModified -** -** The dbModified flag is set whenever a database page is dirtied. -** It is cleared at the end of each transaction. -** -** It is used when committing or otherwise ending a transaction. If -** the dbModified flag is clear then less work has to be done. -** -** journalStarted -** -** This flag is set whenever the the main journal is opened and -** initialized -** -** The point of this flag is that it must be set after the -** first journal header in a journal file has been synced to disk. -** After this has happened, new pages appended to the database -** do not need the PGHDR_NEED_SYNC flag set, as they do not need -** to wait for a journal sync before they can be written out to -** the database file (see function pager_write()). -** ** setMaster ** -** This variable is used to ensure that the master journal file name -** (if any) is only written into the journal file once. +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a master-journal name to be written into the +** journal file before it is synced to disk. ** -** When committing a transaction, the master journal file name (if any) -** may be written into the journal file while the pager is still in -** PAGER_RESERVED state (see CommitPhaseOne() for the action). It -** then attempts to upgrade to an exclusive lock. If this attempt -** fails, then SQLITE_BUSY may be returned to the user and the user -** may attempt to commit the transaction again later (calling -** CommitPhaseOne() again). This flag is used to ensure that the -** master journal name is only written to the journal file the first -** time CommitPhaseOne() is called. +** Whether or not a journal file contains a master-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a master-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a master-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain master journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** master journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setMaster flag +** is cleared anyway (and the pager will move to ERROR state). ** ** doNotSpill, doNotSyncSpill ** -** When enabled, cache spills are prohibited. The doNotSpill variable -** inhibits all cache spill and doNotSyncSpill inhibits those spills that -** would require a journal sync. The doNotSyncSpill is set and cleared -** by sqlite3PagerWrite() in order to prevent a journal sync from happening -** in between the journalling of two pages on the same sector. The -** doNotSpill value set to prevent pagerStress() from trying to use -** the journal during a rollback. -** -** needSync -** -** TODO: It might be easier to set this variable in writeJournalHdr() -** and writeMasterJournal() only. Change its meaning to "unsynced data -** has been written to the journal". +** These two boolean variables control the behaviour of cache-spills +** (calls made by the pcache module to the pagerStress() routine to +** write cached data to the file-system in order to free up memory). +** +** When doNotSpill is non-zero, writing to the database from pagerStress() +** is disabled altogether. This is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). +** +** If doNotSyncSpill is non-zero, writing to the database from pagerStress() +** is permitted, but syncing the journal file is not. This flag is set +** by sqlite3PagerWrite() when the file-system sector-size is larger than +** the database page-size in order to prevent a journal sync from happening +** in between the journalling of two pages on the same sector. ** ** subjInMemory ** ** This is a boolean variable. If true, then any required sub-journal ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ - u8 journalMode; /* On of the PAGER_JOURNALMODE_* values */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ u8 useJournal; /* Use a rollback journal on this file */ u8 noReadlock; /* Do not bother to obtain readlocks */ u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ - u8 sync_flags; /* One of SYNC_NORMAL or SYNC_FULL */ + u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ + u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ u8 tempFile; /* zFilename is a temporary file */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ - /* The following block contains those class members that are dynamically - ** modified during normal operations. The other variables in this structure - ** are either constant throughout the lifetime of the pager, or else - ** used to store configuration parameters that affect the way the pager - ** operates. - ** - ** The 'state' variable is described in more detail along with the - ** descriptions of the values it may take - PAGER_UNLOCK etc. Many of the - ** other variables in this block are described in the comment directly - ** above this class definition. + /************************************************************************** + ** The following block contains those class members that change during + ** routine opertion. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. */ - u8 state; /* PAGER_UNLOCK, _SHARED, _RESERVED, etc. */ - u8 dbModified; /* True if there are any changes to the Db */ - u8 needSync; /* True if an fsync() is needed on the journal */ - u8 journalStarted; /* True if header of journal is synced */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ u8 changeCountDone; /* Set after incrementing the change-counter */ u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 doNotSyncSpill; /* Do not do a spill that requires jrnl sync */ - u8 dbSizeValid; /* Set when dbSize is correct */ u8 subjInMemory; /* True to use in-memory sub-journals */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ int errCode; /* One of several kinds of errors */ int nRec; /* Pages journalled since last j-header written */ u32 cksumInit; /* Quasi-random value added to every checksum */ u32 nSubRec; /* Number of records written to sub-journal */ Bitvec *pInJournal; /* One bit for each page in the database file */ @@ -34043,21 +35343,25 @@ sqlite3_file *fd; /* File descriptor for database */ sqlite3_file *jfd; /* File descriptor for main journal */ sqlite3_file *sjfd; /* File descriptor for sub-journal */ i64 journalOff; /* Current write offset in the journal file */ i64 journalHdr; /* Byte offset to previous journal header */ - i64 journalSizeLimit; /* Size limit for persistent journal files */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - u32 sectorSize; /* Assumed sector size during rollback */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ u16 nExtra; /* Add this many bytes to each in-memory page */ i16 nReserve; /* Number of unused bytes at end of each page */ u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ int pageSize; /* Number of bytes in a page */ Pgno mxPgno; /* Maximum allowed size of the database */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ #ifdef SQLITE_TEST @@ -34071,11 +35375,10 @@ void (*xCodecFree)(void*); /* Destructor for the codec */ void *pCodec; /* First argument to xCodec... methods */ #endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ - sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ #ifndef SQLITE_OMIT_WAL Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ char *zWal; /* File name for write-ahead log */ #endif }; @@ -34150,25 +35453,226 @@ /* ** The maximum legal page number is (2^31 - 1). */ #define PAGER_MAX_PGNO 2147483647 +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods) + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +#ifndef SQLITE_OMIT_WAL +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); +} +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + #ifndef NDEBUG /* ** Usage: ** ** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. */ -static int assert_pager_state(Pager *pPager){ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; - /* A temp-file is always in PAGER_EXCLUSIVE or PAGER_SYNCED state. */ - assert( pPager->tempFile==0 || pPager->state>=PAGER_EXCLUSIVE ); + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); - /* The changeCountDone flag is always set for temp-files */ - assert( pPager->tempFile==0 || pPager->changeCountDone ); + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK || p->noReadlock ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setMaster==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + break; + } return 1; +} +#endif /* ifndef NDEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +*/ +static char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; } #endif /* ** Return true if it is necessary to write page *pPg into the sub-journal. @@ -34217,10 +35721,11 @@ /* ** Write a 32-bit integer into a string buffer in big-endian byte order. */ #define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + /* ** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK ** on success or an error code is something goes wrong. */ @@ -34229,31 +35734,57 @@ put32bits(ac, val); return sqlite3OsWrite(fd, ac, 4, offset); } /* -** The argument to this macro is a file descriptor (type sqlite3_file*). -** Return 0 if it is not open, or non-zero (but not 1) if it is. -** -** This is so that expressions can be written as: -** -** if( isOpen(pPager->jfd) ){ ... -** -** instead of -** -** if( pPager->jfd->pMethods ){ ... -*/ -#define isOpen(pFd) ((pFd)->pMethods) +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. +*/ +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + return rc; +} /* -** If file pFd is open, call sqlite3OsUnlock() on it. +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. */ -static int osUnlock(sqlite3_file *pFd, int eLock){ - if( !isOpen(pFd) ){ - return SQLITE_OK; +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } } - return sqlite3OsUnlock(pFd, eLock); + return rc; } /* ** This function determines whether or not the atomic-write optimization ** can be used with this pager. The optimization can be used if: @@ -34325,17 +35856,18 @@ ** that the page is either dirty or still matches the calculated page-hash. */ #define CHECK_PAGE(x) checkPage(x) static void checkPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; - assert( !pPg->pageHash || pPager->errCode - || (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); } #else #define pager_datahash(X,Y) 0 #define pager_pagehash(X) 0 +#define pager_set_pagehash(X) #define CHECK_PAGE(x) #endif /* SQLITE_CHECK_PAGES */ /* ** When this is called the journal file for pager pPager must be open. @@ -34460,11 +35992,11 @@ }else{ static const char zeroHdr[28] = {0}; rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); } if( rc==SQLITE_OK && !pPager->noSync ){ - rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->sync_flags); + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); } /* At this point the transaction is committed but the write lock ** is still held on the file. If there is a size limit configured for ** the persistent journal and the journal file currently consumes more @@ -34498,11 +36030,11 @@ ** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. */ static int writeJournalHdr(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ - u32 nHeader = pPager->pageSize; /* Size of buffer pointed to by zHeader */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ u32 nWrite; /* Bytes of header sector written */ int ii; /* Loop counter */ assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ @@ -34541,11 +36073,11 @@ ** ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees ** that garbage data is never appended to the journal file. */ assert( isOpen(pPager->fd) || pPager->noSync ); - if( (pPager->noSync) || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); }else{ @@ -34665,18 +36197,25 @@ } if( pPager->journalOff==0 ){ u32 iPageSize; /* Page-size field of journal header */ u32 iSectorSize; /* Sector-size field of journal header */ - u16 iPageSize16; /* Copy of iPageSize in 16-bit variable */ /* Read the page-size and sector-size journal header fields. */ if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) ){ return rc; } + + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power ** of two greater than or equal to 512 or 32, and not greater than their ** respective compile time maximum limits. @@ -34695,14 +36234,12 @@ /* Update the page-size to match the value read from the journal. ** Use a testcase() macro to make sure that malloc failure within ** PagerSetPagesize() is tested. */ - iPageSize16 = (u16)iPageSize; - rc = sqlite3PagerSetPagesize(pPager, &iPageSize16, -1); + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); testcase( rc!=SQLITE_OK ); - assert( rc!=SQLITE_OK || iPageSize16==(u16)iPageSize ); /* Update the assumed sector-size to match the value used by ** the process that created this journal. If this journal was ** created by a process other than this one, then this routine ** is being called from within pager_playback(). The local value @@ -34740,11 +36277,14 @@ int nMaster; /* Length of string zMaster */ i64 iHdrOff; /* Offset of header in journal file */ i64 jrnlSize; /* Size of journal file on disk */ u32 cksum = 0; /* Checksum of string zMaster */ - if( !zMaster || pPager->setMaster + assert( pPager->setMaster==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->journalMode==PAGER_JOURNALMODE_OFF ){ return SQLITE_OK; } @@ -34776,11 +36316,10 @@ || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, iHdrOff+4+nMaster+8))) ){ return rc; } pPager->journalOff += (nMaster+20); - pPager->needSync = !pPager->noSync; /* If the pager is in peristent-journal mode, then the physical ** journal-file may extend past the end of the master-journal name ** and 8 bytes of magic data just written to the file. This is ** dangerous because the code to rollback a hot-journal file @@ -34812,21 +36351,15 @@ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); return p; } /* -** Unless the pager is in error-state, discard all in-memory pages. If -** the pager is in error-state, then this call is a no-op. -** -** TODO: Why can we not reset the pager while in error state? +** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ - if( SQLITE_OK==pPager->errCode ){ - sqlite3BackupRestart(pPager->pBackup); - sqlite3PcacheClear(pPager->pPCache); - pPager->dbSizeValid = 0; - } + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); } /* ** Free all structures in the Pager.aSavepoint[] array and set both ** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal @@ -34865,38 +36398,43 @@ } return rc; } /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. -*/ -#ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); -} -#else -# define pagerUseWal(x) 0 -# define pagerRollbackWal(x) 0 -# define pagerWalFrames(v,w,x,y,z) 0 -# define pagerOpenWalIfPresent(z) SQLITE_OK -# define pagerBeginReadTransaction(z) SQLITE_OK -#endif - -/* -** Unlock the database file. This function is a no-op if the pager -** is in exclusive mode. +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. ** -** If the pager is currently in error state, discard the contents of -** the cache and reset the Pager structure internal state. If there is -** an open journal-file, then the next time a shared-lock is obtained -** on the pager file (by this or any other process), it will be -** treated as a hot-journal and rolled back. +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). */ static void pager_unlock(Pager *pPager){ - if( !pPager->exclusiveMode ){ - int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; /* If the operating system support deletion of open files, then ** close the journal file when dropping the database lock. Otherwise ** another connection with journal_mode=delete might delete the file @@ -34912,62 +36450,60 @@ || 1!=(pPager->journalMode & 5) ){ sqlite3OsClose(pPager->jfd); } - sqlite3BitvecDestroy(pPager->pInJournal); - pPager->pInJournal = 0; - releaseAllSavepoints(pPager); - - /* If the file is unlocked, somebody else might change it. The - ** values stored in Pager.dbSize etc. might become invalid if - ** this happens. One can argue that this doesn't need to be cleared - ** until the change-counter check fails in PagerSharedLock(). - ** Clearing the page size cache here is being conservative. - */ - pPager->dbSizeValid = 0; - - if( pagerUseWal(pPager) ){ - sqlite3WalEndReadTransaction(pPager->pWal); - }else{ - rc = osUnlock(pPager->fd, NO_LOCK); - } - if( rc ){ - pPager->errCode = rc; - } - IOTRACE(("UNLOCK %p\n", pPager)) - - /* If Pager.errCode is set, the contents of the pager cache cannot be - ** trusted. Now that the pager file is unlocked, the contents of the - ** cache can be discarded and the error code safely cleared. - */ - if( pPager->errCode ){ - if( rc==SQLITE_OK ){ - pPager->errCode = SQLITE_OK; - } - pager_reset(pPager); - } - + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } + + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); pPager->changeCountDone = 0; - pPager->state = PAGER_UNLOCK; - pPager->dbModified = 0; + pPager->eState = PAGER_OPEN; } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + if( pPager->errCode ){ + assert( !MEMDB ); + pager_reset(pPager); + pPager->changeCountDone = pPager->tempFile; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setMaster = 0; } /* -** This function should be called when an IOERR, CORRUPT or FULL error -** may have occurred. The first argument is a pointer to the pager -** structure, the second the error-code about to be returned by a pager -** API function. The value returned is a copy of the second argument -** to this function. -** -** If the second argument is SQLITE_IOERR, SQLITE_CORRUPT, or SQLITE_FULL -** the error becomes persistent. Until the persistent error is cleared, -** subsequent API calls on this Pager will immediately return the same -** error code. -** -** A persistent error indicates that the contents of the pager-cache +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. +** +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. +** +** The ERROR state indicates that the contents of the pager-cache ** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need ** to be replayed to restore the contents of the database file (as if ** it were a hot-journal). @@ -34980,49 +36516,25 @@ pPager->errCode==SQLITE_OK || (pPager->errCode & 0xff)==SQLITE_IOERR ); if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; + pPager->eState = PAGER_ERROR; } return rc; } -/* -** Execute a rollback if a transaction is active and unlock the -** database file. -** -** If the pager has already entered the error state, do not attempt -** the rollback at this time. Instead, pager_unlock() is called. The -** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and clear the error state. If this means that -** there is a hot-journal left in the file-system, the next connection -** to obtain a shared lock on the pager (which may be this one) will -** roll it back. -** -** If the pager has not already entered the error state, but an IO or -** malloc error occurs during a rollback, then this will itself cause -** the pager to enter the error state. Which will be cleared by the -** call to pager_unlock(), as described above. -*/ -static void pagerUnlockAndRollback(Pager *pPager){ - if( pPager->errCode==SQLITE_OK && pPager->state>=PAGER_RESERVED ){ - sqlite3BeginBenignMalloc(); - sqlite3PagerRollback(pPager); - sqlite3EndBenignMalloc(); - } - pager_unlock(pPager); -} - /* ** This routine ends a transaction. A transaction is usually ended by ** either a COMMIT or a ROLLBACK operation. This routine may be called ** after rollback of a hot-journal, or if an error occurs while opening ** the journal file or writing the very first journal-header of a ** database transaction. ** -** If the pager is in PAGER_SHARED or PAGER_UNLOCK state when this -** routine is called, it is a no-op (returns SQLITE_OK). +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** ** If the journal file is open, then it is "finalized". Once a journal ** file has been finalized it is not possible to use it to roll back a @@ -35049,17 +36561,13 @@ ** If the pager is running in exclusive mode, this method of finalizing ** the journal file is never used. Instead, if the journalMode is ** DELETE and the pager is in exclusive mode, the method described under ** journalMode==PERSIST is used instead. ** -** After the journal is finalized, if running in non-exclusive mode, the -** pager moves to PAGER_SHARED state (and downgrades the lock on the -** database file accordingly). -** -** If the pager is running in exclusive mode and is in PAGER_SYNCED state, -** it moves to PAGER_EXCLUSIVE. No locks are downgraded when running in -** exclusive mode. +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the ** database then the IO error code is returned to the user. If the ** operation to finalize the journal file fails, then the code still @@ -35070,15 +36578,30 @@ */ static int pager_end_transaction(Pager *pPager, int hasMaster){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ - if( pPager->stateeState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ @@ -35090,18 +36613,15 @@ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); } pPager->journalOff = 0; - pPager->journalStarted = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ rc = zeroJournalHdr(pPager, hasMaster); - pager_error(pPager, rc); pPager->journalOff = 0; - pPager->journalStarted = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to ** the database file, it will do so using an in-memory journal. @@ -35113,52 +36633,80 @@ sqlite3OsClose(pPager->jfd); if( !pPager->tempFile ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } } + } #ifdef SQLITE_CHECK_PAGES - sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = pager_lookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnref(p); + } + } #endif - } + sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); assert( rc2==SQLITE_OK ); - pPager->state = PAGER_SHARED; - - /* If the connection was in locking_mode=exclusive mode but is no longer, - ** drop the EXCLUSIVE lock held on the database file. - */ - if( !pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, 0) ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - } - }else if( !pPager->exclusiveMode ){ - rc2 = osUnlock(pPager->fd, SHARED_LOCK); - pPager->state = PAGER_SHARED; + } + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; - }else if( pPager->state==PAGER_SYNCED ){ - pPager->state = PAGER_EXCLUSIVE; - } - pPager->setMaster = 0; - pPager->needSync = 0; - pPager->dbModified = 0; - - /* TODO: Is this optimal? Why is the db size invalidated here - ** when the database file is not unlocked? */ - pPager->dbOrigSize = 0; - sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } + } + pPager->eState = PAGER_READER; + pPager->setMaster = 0; return (rc==SQLITE_OK?rc2:rc); } + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0); + } + } + pager_unlock(pPager); +} /* ** Parameter aData must point to a buffer of pPager->pageSize bytes ** of data. Compute and return a checksum based ont the contents of the ** page of data and the current value of pPager->cksumInit. @@ -35206,13 +36754,12 @@ ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The isMainJrnl flag is true if this is the main rollback journal and -** false for the statement journal. The main rollback journal uses -** checksums - the statement journal does not. +** The main rollback journal uses checksums - the statement journal does +** not. ** ** If the page number of the page record read from the (sub-)journal file ** is greater than the current value of Pager.dbSize, then playback is ** skipped and SQLITE_OK is returned. ** @@ -35261,10 +36808,21 @@ assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ aData = pPager->pTmpSpace; assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. */ jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; @@ -35298,20 +36856,19 @@ ** rollback, then don't bother to play it back again. */ if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ return rc; } - assert( pPager->state==PAGER_RESERVED || pPager->state>=PAGER_EXCLUSIVE ); /* When playing back page 1, restore the nReserve setting */ if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ pPager->nReserve = ((u8*)aData)[20]; pagerReportSize(pPager); } - /* If the pager is in RESERVED state, then there must be a copy of this + /* If the pager is in CACHEMOD state, then there must be a copy of this ** page in the pager cache. In this case just update the pager cache, ** not the database file. The page is left marked dirty in this case. ** ** An exception to the above rule: If the database is in no-sync mode ** and a page is moved during an incremental vacuum then the page may @@ -35318,12 +36875,15 @@ ** not be in the pager cache. Later: if a malloc() or IO error occurs ** during a Movepage() call, then the page may not be in the cache ** either. So the condition described in the above paragraph is not ** assert()able. ** - ** If in EXCLUSIVE state, then we update the pager cache if it exists - ** and the main file. The page is then marked not dirty. + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. ** ** Ticket #1171: The statement journal might contain page content that is ** different from the page content at the start of the transaction. ** This occurs when a page is changed prior to the start of a statement ** then changed again within the statement. When rolling back such a @@ -35345,21 +36905,22 @@ pPg = 0; }else{ pPg = pager_lookup(pPager, pgno); } assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), (isMainJrnl?"main-journal":"sub-journal") )); if( isMainJrnl ){ isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); }else{ isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); } - if( (pPager->state>=PAGER_EXCLUSIVE) - && isOpen(pPager->fd) + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) && isSynced ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); @@ -35431,13 +36992,12 @@ ** database corruption may ensue. */ assert( !pagerUseWal(pPager) ); sqlite3PcacheMakeClean(pPg); } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); + /* If this was page 1, then restore the value of Pager.dbFileVers. ** Do this before any decoding. */ if( pgno==1 ){ memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); } @@ -35585,14 +37145,14 @@ /* ** This function is used to change the actual size of the database ** file in the file-system. This only happens when committing a transaction, ** or rolling back a transaction (including rolling back a hot-journal). ** -** If the main database file is not open, or an exclusive lock is not -** held, this function is a no-op. Otherwise, the size of the file is -** changed to nPage pages (nPage*pPager->pageSize bytes). If the file -** on disk is currently larger than nPage pages, then use the VFS +** If the main database file is not open, or the pager is not in either +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** ** Or, it might might be the case that the file on disk is smaller than ** nPage pages. Some operating system implementations can get confused if ** you try to truncate a file to some size that is larger than it @@ -35602,12 +37162,18 @@ ** If successful, return SQLITE_OK. If an IO error occurs while modifying ** the database file, return the error code to the caller. */ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; - if( pPager->state>=PAGER_EXCLUSIVE && isOpen(pPager->fd) ){ + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ i64 currentSize, newSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* TODO: Is it safe to use Pager.dbFileSize here? */ rc = sqlite3OsFileSize(pPager->fd, ¤tSize); newSize = pPager->pageSize*(i64)nPage; if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ @@ -35727,11 +37293,11 @@ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. */ assert( isOpen(pPager->jfd) ); rc = sqlite3OsFileSize(pPager->jfd, &szJ); - if( rc!=SQLITE_OK || szJ==0 ){ + if( rc!=SQLITE_OK ){ goto end_playback; } /* Read the master journal name from the journal, if it is present. ** If a master journal file name is specified, but the file is not @@ -35761,11 +37327,11 @@ ** occurs. */ while( 1 ){ /* Read the next journal header from the journal file. If there are ** not enough bytes left in the journal file for a complete header, or - ** it is corrupted, then a process must of failed while writing it. + ** it is corrupted, then a process must have failed while writing it. ** This indicates nothing more needs to be rolled back. */ rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ @@ -35875,15 +37441,14 @@ if( rc==SQLITE_OK ){ zMaster = pPager->pTmpSpace; rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); - } - if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + if( rc==SQLITE_OK && !pPager->noSync + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } if( rc==SQLITE_OK ){ rc = pager_end_transaction(pPager, zMaster[0]!='\0'); testcase( rc!=SQLITE_OK ); } @@ -35920,11 +37485,11 @@ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ - assert( pPager->state>=PAGER_SHARED && !MEMDB ); + assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); if( NEVER(!isOpen(pPager->fd)) ){ assert( pPager->tempFile ); memset(pPg->pData, 0, pPager->pageSize); @@ -36053,24 +37618,32 @@ static int pagerWalFrames( Pager *pPager, /* Pager object */ PgHdr *pList, /* List of frames to log */ Pgno nTruncate, /* Database size after this commit */ int isCommit, /* True if this is a commit */ - int sync_flags /* Flags to pass to OsSync() (or 0) */ + int syncFlags /* Flags to pass to OsSync() (or 0) */ ){ int rc; /* Return code */ assert( pPager->pWal ); rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, nTruncate, isCommit, sync_flags + pPager->pageSize, pList, nTruncate, isCommit, syncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ PgHdr *p; for(p=pList; p; p=p->pDirty){ sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); } } + +#ifdef SQLITE_CHECK_PAGES + { + PgHdr *p; + for(p=pList; p; p=p->pDirty) pager_set_pagehash(p); + } +#endif + return rc; } /* ** Begin a read transaction on the WAL. @@ -36083,32 +37656,84 @@ static int pagerBeginReadTransaction(Pager *pPager){ int rc; /* Return code */ int changed = 0; /* True if cache must be reset */ assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); /* sqlite3WalEndReadTransaction() was not called for the previous ** transaction in locking_mode=EXCLUSIVE. So call it now. If we ** are in locking_mode=NORMAL and EndRead() was previously called, ** the duplicate call is harmless. */ sqlite3WalEndReadTransaction(pPager->pWal); rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); - } - pPager->state = PAGER_SHARED; + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); + } return rc; } +#endif +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the database size was not available from the WAL sub-system, + ** determine it based on the size of the database file. If the size + ** of the database file is not an integer multiple of the page-size, + ** round down to the nearest page. Except, any file larger than 0 + ** bytes in size is considered to contain at least one page. + */ + if( nPage==0 ){ + i64 n = 0; /* Size of db file in bytes */ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + } + nPage = (Pgno)(n / pPager->pageSize); + if( nPage==0 && n>0 ){ + nPage = 1; + } + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager ** exists if the database is not empy, or verify that the *-wal file does ** not exist (by deleting it) if the database file is empty. ** @@ -36117,25 +37742,26 @@ ** if no error occurs, make sure Pager.journalMode is not set to ** PAGER_JOURNALMODE_WAL. ** ** Return SQLITE_OK or an error code. ** -** If the WAL file is opened, also open a snapshot (read transaction). -** ** The caller must hold a SHARED lock on the database file to call this ** function. Because an EXCLUSIVE lock on the db file is required to delete ** a WAL on a none-empty database, this ensures there is no race condition ** between the xAccess() below and an xDelete() being executed by some ** other connection. */ static int pagerOpenWalIfPresent(Pager *pPager){ int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK || pPager->noReadlock ); + if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ - int nPage; /* Size of the database file */ - assert( pPager->state>=SHARED_LOCK ); - rc = sqlite3PagerPagecount(pPager, &nPage); + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); if( rc ) return rc; if( nPage==0 ){ rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); isWal = 0; }else{ @@ -36143,15 +37769,12 @@ pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal ); } if( rc==SQLITE_OK ){ if( isWal ){ - pager_reset(pPager); + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); rc = sqlite3PagerOpenWal(pPager, 0); - if( rc==SQLITE_OK ){ - rc = pagerBeginReadTransaction(pPager); - } }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } } } @@ -36199,11 +37822,12 @@ i64 szJ; /* Effective size of the main journal */ i64 iHdrOff; /* End of first segment of main-journal records */ int rc = SQLITE_OK; /* Return code */ Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ - assert( pPager->state>=PAGER_SHARED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); /* Allocate a bitvec to use to store the set of pages rolled back */ if( pSavepoint ){ pDone = sqlite3BitvecCreate(pSavepoint->nOrig); if( !pDone ){ @@ -36329,20 +37953,54 @@ ** of the journal header - being written in between the two ** syncs). If we assume that writing a ** single disk sector is atomic, then this mode provides ** assurance that the journal will not be corrupted to the ** point of causing damage to the database during rollback. +** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. ** ** Numeric values associated with these states are OFF==1, NORMAL=2, ** and FULL=3. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){ +SQLITE_PRIVATE void sqlite3PagerSetSafetyLevel( + Pager *pPager, /* The pager to set safety level for */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int bFullFsync, /* PRAGMA fullfsync */ + int bCkptFullFsync /* PRAGMA checkpoint_fullfsync */ +){ + assert( level>=1 && level<=3 ); pPager->noSync = (level==1 || pPager->tempFile) ?1:0; pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0; - pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL); - if( pPager->noSync ) pPager->needSync = 0; + if( pPager->noSync ){ + pPager->syncFlags = 0; + pPager->ckptSyncFlags = 0; + }else if( bFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else if( bCkptFullFsync ){ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; + } } #endif /* ** The following global variable is incremented whenever the library @@ -36420,11 +38078,11 @@ ** Change the page size used by the Pager object. The new page size ** is passed in *pPageSize. ** ** If the pager is in the error state when this function is called, it ** is a no-op. The value returned is the error state error code (i.e. -** one of SQLITE_IOERR, SQLITE_CORRUPT or SQLITE_FULL). +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). ** ** Otherwise, if all of the following are true: ** ** * the new page size (value of *pPageSize) is valid (a power ** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and @@ -36444,32 +38102,52 @@ ** If the page size is not changed, either because one of the enumerated ** conditions above is not true, the pager was in error state when this ** function was called, or because the memory allocation attempt failed, ** then *pPageSize is set to the old, retained page size before returning. */ -SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u16 *pPageSize, int nReserve){ - int rc = pPager->errCode; - - if( rc==SQLITE_OK ){ - u16 pageSize = *pPageSize; - assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); - if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=pPager->pageSize - ){ - char *pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ){ - rc = SQLITE_NOMEM; - }else{ - pager_reset(pPager); - pPager->pageSize = pageSize; - sqlite3PageFree(pPager->pTmpSpace); - pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); - } - } - *pPageSize = (u16)pPager->pageSize; +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; + + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + } + if( rc==SQLITE_OK ){ + pNew = (char *)sqlite3PageMalloc(pageSize); + if( !pNew ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + pPager->dbSize = (Pgno)(nByte/pageSize); + pPager->pageSize = pageSize; + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ if( nReserve<0 ) nReserve = pPager->nReserve; assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); } @@ -36494,18 +38172,15 @@ ** maximum page count below the current size of the database. ** ** Regardless of mxPage, return the current maximum page count. */ SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ - int nPage; if( mxPage>0 ){ pPager->mxPgno = mxPage; } - if( pPager->state!=PAGER_UNLOCK ){ - sqlite3PagerPagecount(pPager, &nPage); - assert( (int)pPager->mxPgno>=nPage ); - } + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ return pPager->mxPgno; } /* ** The following set of routines are used to disable the simulated @@ -36565,70 +38240,20 @@ } return rc; } /* -** Return the total number of pages in the database file associated -** with pPager. Normally, this is calculated as (/). +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** ** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. -** -** If the pager is in error state when this function is called, then the -** error state error code is returned and *pnPage left unchanged. Or, -** if the file system has to be queried for the size of the file and -** the query attempt returns an IO error, the IO error code is returned -** and *pnPage is left unchanged. -** -** Otherwise, if everything is successful, then SQLITE_OK is returned -** and *pnPage is set to the number of pages in the database. -*/ -SQLITE_PRIVATE int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ - Pgno nPage = 0; /* Value to return via *pnPage */ - - /* Determine the number of pages in the file. Store this in nPage. */ - if( pPager->dbSizeValid ){ - nPage = pPager->dbSize; - }else{ - int rc; /* Error returned by OsFileSize() */ - i64 n = 0; /* File size in bytes returned by OsFileSize() */ - - if( pagerUseWal(pPager) && pPager->state!=PAGER_UNLOCK ){ - sqlite3WalDbsize(pPager->pWal, &nPage); - } - - if( nPage==0 ){ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) ){ - if( SQLITE_OK!=(rc = sqlite3OsFileSize(pPager->fd, &n)) ){ - pager_error(pPager, rc); - return rc; - } - } - if( n>0 && npageSize ){ - nPage = 1; - }else{ - nPage = (Pgno)(n / pPager->pageSize); - } - } - if( pPager->state!=PAGER_UNLOCK ){ - pPager->dbSize = nPage; - pPager->dbFileSize = nPage; - pPager->dbSizeValid = 1; - } - } - - /* If the current number of pages in the file is greater than the - ** configured maximum pager number, increase the allowed limit so - ** that the file can be read. - */ - if( nPage>pPager->mxPgno ){ - pPager->mxPgno = (Pgno)nPage; - } - - /* Set the output variable and return SQLITE_OK */ - *pnPage = nPage; - return SQLITE_OK; +*/ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; } /* ** Try to obtain a lock of type locktype on the database file. If @@ -36645,42 +38270,23 @@ ** variable to locktype before returning. */ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* The OS lock values must be the same as the Pager lock values */ - assert( PAGER_SHARED==SHARED_LOCK ); - assert( PAGER_RESERVED==RESERVED_LOCK ); - assert( PAGER_EXCLUSIVE==EXCLUSIVE_LOCK ); - - /* If the file is currently unlocked then the size must be unknown. It - ** must not have been modified at this point. - */ - assert( pPager->state>=PAGER_SHARED || pPager->dbSizeValid==0 ); - assert( pPager->state>=PAGER_SHARED || pPager->dbModified==0 ); - /* Check that this is either a no-op (because the requested lock is ** already held, or one of the transistions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ - assert( (pPager->state>=locktype) - || (pPager->state==PAGER_UNLOCK && locktype==PAGER_SHARED) - || (pPager->state==PAGER_RESERVED && locktype==PAGER_EXCLUSIVE) + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); - if( pPager->state>=locktype ){ - rc = SQLITE_OK; - }else{ - do { - rc = sqlite3OsLock(pPager->fd, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); - if( rc==SQLITE_OK ){ - pPager->state = (u8)locktype; - IOTRACE(("LOCK %p %d\n", pPager, locktype)) - } - } + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); return rc; } /* ** Function assertTruncateConstraint(pPager) checks that one of the @@ -36721,13 +38327,12 @@ ** function does not actually modify the database file on disk. It ** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSizeValid ); assert( pPager->dbSize>=nPage ); - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; assertTruncateConstraint(pPager); } @@ -36773,31 +38378,33 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ u8 *pTmp = (u8 *)pPager->pTmpSpace; disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pPager->errCode = 0; + /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, pTmp - ); + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); pPager->pWal = 0; #endif pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); }else{ - /* Set Pager.journalHdr to -1 for the benefit of the pager_playback() - ** call which may be made from within pagerUnlockAndRollback(). If it - ** is not -1, then the unsynced portion of an open journal file may - ** be played back into the database. If a power failure occurs while - ** this is happening, the database may become corrupt. + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. */ if( isOpen(pPager->jfd) ){ - pPager->errCode = pagerSyncHotJournal(pPager); + pager_error(pPager, pagerSyncHotJournal(pPager)); } pagerUnlockAndRollback(pPager); } sqlite3EndBenignMalloc(); enable_simulated_io_errors(); @@ -36838,13 +38445,13 @@ /* ** Sync the journal. In other words, make sure all the pages that have ** been written to the journal have actually reached the surface of the ** disk and can be restored in the event of a hot-journal rollback. ** -** If the Pager.needSync flag is not set, then this function is a -** no-op. Otherwise, the actions required depend on the journal-mode -** and the device characteristics of the the file-system, as follows: +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need ** be taken. ** ** * Otherwise, if the device does not support the SAFE_APPEND property, @@ -36864,22 +38471,29 @@ ** ** } ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** The Pager.needSync flag is never be set for temporary files, or any -** file operating in no-sync mode (Pager.noSync set to non-zero). -** ** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ -static int syncJournal(Pager *pPager){ - if( pPager->needSync ){ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ assert( !pPager->tempFile ); - if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ - int rc; /* Return code */ + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); assert( isOpen(pPager->jfd) ); if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ /* This block deals with an obscure problem. If the last connection @@ -36933,11 +38547,11 @@ ** and never needs to be updated. */ if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); if( rc!=SQLITE_OK ) return rc; } IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); rc = sqlite3OsWrite( pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr @@ -36945,10 +38559,17 @@ if( rc!=SQLITE_OK ) return rc; } if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->sync_flags| - (pPager->sync_flags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) ); if( rc!=SQLITE_OK ) return rc; } + + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } @@ -36955,16 +38576,17 @@ - } - - /* The journal file was just successfully synced. Set Pager.needSync - ** to zero and clear the PGHDR_NEED_SYNC flag on all pagess. - */ - pPager->needSync = 0; - pPager->journalStarted = 1; - pPager->journalHdr = pPager->journalOff; - sqlite3PcacheClearSyncFlags(pPager->pPCache); + }else{ + pPager->journalHdr = pPager->journalOff; + } } + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); return SQLITE_OK; } /* ** The argument is the first in a linked list of dirty pages connected @@ -36997,31 +38619,16 @@ ** If everything is successful, SQLITE_OK is returned. If an IO error ** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot ** be obtained, SQLITE_BUSY is returned. */ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ - int rc; /* Return code */ - - /* At this point there may be either a RESERVED or EXCLUSIVE lock on the - ** database file. If there is already an EXCLUSIVE lock, the following - ** call is a no-op. - ** - ** Moving the lock from RESERVED to EXCLUSIVE actually involves going - ** through an intermediate state PENDING. A PENDING lock prevents new - ** readers from attaching to the database but is unsufficient for us to - ** write. The idea of a PENDING lock is to prevent new readers from - ** coming in while we wait for existing readers to clear. - ** - ** While the pager is in the RESERVED state, the original database file - ** is unchanged and we can rollback without having to playback the - ** journal into the original database file. Once we transition to - ** EXCLUSIVE, it means the database file has been changed and any rollback - ** will require a journal playback. - */ + int rc = SQLITE_OK; /* Return code */ + + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ assert( !pagerUseWal(pPager) ); - assert( pPager->state>=PAGER_RESERVED ); - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); /* If the file is a temp-file has not yet been opened, open it now. It ** is not possible for rc to be other than SQLITE_OK if this branch ** is taken, as pager_wait_on_lock() is a no-op for temp-files. */ @@ -37032,13 +38639,14 @@ /* Before the first write, give the VFS a hint of what the final ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>(pPager->dbOrigSize+1) ){ + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; } while( rc==SQLITE_OK && pList ){ Pgno pgno = pList->pgno; @@ -37051,10 +38659,12 @@ ** set (set by sqlite3PagerDontWrite()). */ if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ char *pData; /* Data to write */ + + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); /* Encode the database */ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ @@ -37080,13 +38690,11 @@ PAGER_INCR(sqlite3_pager_writedb_count); PAGER_INCR(pPager->nWrite); }else{ PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); } -#ifdef SQLITE_CHECK_PAGES - pList->pageHash = pager_pagehash(pList); -#endif + pager_set_pagehash(pList); pList = pList->pDirty; } return rc; } @@ -37194,13 +38802,18 @@ ** pages belonging to the same sector. ** ** The doNotSpill flag inhibits all cache spilling regardless of whether ** or not a sync is required. This is set during a rollback. ** - ** Spilling is also inhibited when in an error state. + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementaton it + ** is impossible for sqlite3PCacheFetch() to be called with createFlag==1 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. */ - if( pPager->errCode ) return SQLITE_OK; + if( NEVER(pPager->errCode) ) return SQLITE_OK; if( pPager->doNotSpill ) return SQLITE_OK; if( pPager->doNotSyncSpill && (pPg->flags & PGHDR_NEED_SYNC)!=0 ){ return SQLITE_OK; } @@ -37214,20 +38827,14 @@ rc = pagerWalFrames(pPager, pPg, 0, 0, 0); } }else{ /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC ){ - assert( !pPager->noSync ); - rc = syncJournal(pPager); - if( rc==SQLITE_OK && - !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && - !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) - ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); - } + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); } /* If the page number of this page is larger than the current size of ** the database image, it may need to be written to the sub-journal. ** This is because the call to pager_write_pagelist() below will not @@ -37261,10 +38868,11 @@ rc = subjournalPage(pPg); } /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); rc = pager_write_pagelist(pPager, pPg); } } /* Mark the page as clean. */ @@ -37271,11 +38879,11 @@ if( rc==SQLITE_OK ){ PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); sqlite3PcacheMakeClean(pPg); } - return pager_error(pPager, rc); + return pager_error(pPager, rc); } /* ** Allocate and initialize a new Pager object and put a pointer to it @@ -37326,11 +38934,11 @@ char *zPathname = 0; /* Full path to database file */ int nPathname = 0; /* Number of bytes in zPathname */ int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ int noReadlock = (flags & PAGER_NO_READLOCK)!=0; /* True to omit read-lock */ int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ - u16 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). This ** is the maximum space required for an in-memory journal file handle ** and a regular journal file-handle. Note that a "regular journal-handle" @@ -37344,10 +38952,17 @@ journalFileSize = ROUND8(sqlite3MemJournalSize()); } /* Set the output variable to NULL in case an error occurs. */ *ppPager = 0; + +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + zFilename = 0; + } +#endif /* Compute and store the full pathname in an allocated buffer pointed ** to by zPathname, length nPathname. Or, if this is a temporary file, ** leave both nPathname and zPathname set to 0. */ @@ -37355,21 +38970,12 @@ nPathname = pVfs->mxPathname+1; zPathname = sqlite3Malloc(nPathname*2); if( zPathname==0 ){ return SQLITE_NOMEM; } -#ifndef SQLITE_OMIT_MEMORYDB - if( strcmp(zFilename,":memory:")==0 ){ - memDb = 1; - zPathname[0] = 0; - }else -#endif - { - zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ - rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); - } - + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); nPathname = sqlite3Strlen30(zPathname); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname ** bytes in length. This means the database cannot be opened, @@ -37420,34 +39026,31 @@ pPager->zFilename = (char*)(pPtr += journalFileSize); assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ if( zPathname ){ + assert( nPathname>0 ); pPager->zJournal = (char*)(pPtr += nPathname + 1); memcpy(pPager->zFilename, zPathname, nPathname); memcpy(pPager->zJournal, zPathname, nPathname); memcpy(&pPager->zJournal[nPathname], "-journal", 8); - if( pPager->zFilename[0]==0 ){ - pPager->zJournal[0] = 0; - } #ifndef SQLITE_OMIT_WAL - else{ - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal", 4); - } + pPager->zWal = &pPager->zJournal[nPathname+8+1]; + memcpy(pPager->zWal, zPathname, nPathname); + memcpy(&pPager->zWal[nPathname], "-wal", 4); #endif sqlite3_free(zPathname); } pPager->pVfs = pVfs; pPager->vfsFlags = vfsFlags; /* Open the pager file. */ - if( zFilename && zFilename[0] && !memDb ){ + if( zFilename && zFilename[0] ){ int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); readOnly = (fout&SQLITE_OPEN_READONLY); /* If the file was successfully opened for read/write access, ** choose a default page size in case we have to create the ** database file. The default page size is the maximum of: @@ -37461,11 +39064,11 @@ assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); if( szPageDfltsectorSize ){ if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; }else{ - szPageDflt = (u16)pPager->sectorSize; + szPageDflt = (u32)pPager->sectorSize; } } #ifdef SQLITE_ENABLE_ATOMIC_WRITE { int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); @@ -37489,11 +39092,12 @@ ** This branch is also run for an in-memory database. An in-memory ** database is the same as a temp-file that is never written out to ** disk and uses an in-memory rollback journal. */ tempFile = 1; - pPager->state = PAGER_EXCLUSIVE; + pPager->eState = PAGER_READER; + pPager->eLock = EXCLUSIVE_LOCK; readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } /* The following call to PagerSetPagesize() serves to set the value of ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. @@ -37526,31 +39130,32 @@ pPager->useJournal = (u8)useJournal; pPager->noReadlock = (noReadlock && readOnly) ?1:0; /* pPager->stmtOpen = 0; */ /* pPager->stmtInUse = 0; */ /* pPager->nRef = 0; */ - pPager->dbSizeValid = (u8)memDb; /* pPager->stmtSize = 0; */ /* pPager->stmtJSize = 0; */ /* pPager->nPage = 0; */ pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; /* pPager->state = PAGER_UNLOCK; */ +#if 0 assert( pPager->state == (tempFile ? PAGER_EXCLUSIVE : PAGER_UNLOCK) ); +#endif /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; assert( tempFile==PAGER_LOCKINGMODE_NORMAL || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); pPager->exclusiveMode = (u8)tempFile; pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; - /* pPager->needSync = 0; */ assert( useJournal || pPager->tempFile ); pPager->noSync = pPager->tempFile; pPager->fullSync = pPager->noSync ?0:1; - pPager->sync_flags = SQLITE_SYNC_NORMAL; + pPager->syncFlags = pPager->noSync ? 0 : SQLITE_SYNC_NORMAL; + pPager->ckptSyncFlags = pPager->syncFlags; /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ pPager->nExtra = (u16)nExtra; pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; @@ -37607,24 +39212,24 @@ sqlite3_vfs * const pVfs = pPager->pVfs; int rc = SQLITE_OK; /* Return code */ int exists = 1; /* True if a journal file is present */ int jrnlOpen = !!isOpen(pPager->jfd); - assert( pPager!=0 ); assert( pPager->useJournal ); assert( isOpen(pPager->fd) ); - assert( pPager->state <= PAGER_SHARED ); + assert( pPager->eState==PAGER_OPEN ); + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN )); *pExists = 0; if( !jrnlOpen ){ rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); } if( rc==SQLITE_OK && exists ){ - int locked; /* True if some process holds a RESERVED lock */ + int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the ** the RESERVED lock and have a journal open at the sqlite3OsAccess() ** call above, but then delete the journal and drop the lock before ** we get to the following sqlite3OsCheckReservedLock() call. If that @@ -37632,25 +39237,25 @@ ** in fact there is none. This results in a false-positive which will ** be dealt with by the playback routine. Ticket #3883. */ rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); if( rc==SQLITE_OK && !locked ){ - int nPage; + Pgno nPage; /* Number of pages in database file */ /* Check the size of the database file. If it consists of 0 pages, ** then delete the journal file. See the header comment above for ** the reasoning here. Delete the obsolete journal file under ** a RESERVED lock to avoid race conditions and to avoid violating ** [H33020]. */ - rc = sqlite3PagerPagecount(pPager, &nPage); + rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ if( nPage==0 ){ sqlite3BeginBenignMalloc(); - if( sqlite3OsLock(pPager->fd, RESERVED_LOCK)==SQLITE_OK ){ + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); } sqlite3EndBenignMalloc(); }else{ /* The journal file exists and no other connection has a reserved ** or greater lock on the database file. Now check that there is @@ -37699,11 +39304,11 @@ ** has been successfully called. If a shared-lock is already held when ** this function is called, it is a no-op. ** ** The following operations are also performed by this function. ** -** 1) If the pager is currently in PAGER_UNLOCK state (no lock held +** 1) If the pager is currently in PAGER_OPEN state (no lock held ** on the database file), then an attempt is made to obtain a ** SHARED lock on the database file. Immediately after obtaining ** the SHARED lock, the file-system is checked for a hot-journal, ** which is played back if present. Following any hot-journal ** rollback, the contents of the cache are validated by checking @@ -37714,70 +39319,51 @@ ** no outstanding references to any pages, and is in the error state, ** then an attempt is made to clear the error state by discarding ** the contents of the page cache and rolling back any open journal ** file. ** -** If the operation described by (2) above is not attempted, and if the -** pager is in an error state other than SQLITE_FULL when this is called, -** the error state error code is returned. It is permitted to read the -** database when in SQLITE_FULL error state. -** -** Otherwise, if everything is successful, SQLITE_OK is returned. If an -** IO error occurs while locking the database, checking for a hot-journal -** file or rolling back a journal file, the IO error code is returned. +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. */ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int isErrorReset = 0; /* True if recovering from error state */ /* This routine is only called from b-tree and only when there are no - ** outstanding pages */ + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. + */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); if( NEVER(MEMDB && pPager->errCode) ){ return pPager->errCode; } - /* If this database is in an error-state, now is a chance to clear - ** the error. Discard the contents of the pager-cache and rollback - ** any hot journal in the file-system. - */ - if( pPager->errCode ){ - if( isOpen(pPager->jfd) || pPager->zJournal ){ - isErrorReset = 1; - } - pPager->errCode = SQLITE_OK; - pager_reset(pPager); - } - - if( pagerUseWal(pPager) ){ - rc = pagerBeginReadTransaction(pPager); - }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ - sqlite3_vfs * const pVfs = pPager->pVfs; - int isHotJournal = 0; + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ + assert( !MEMDB ); - assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ + assert( pPager->noReadlock==0 || pPager->readOnly ); + + if( pPager->noReadlock==0 ){ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; } } - assert( pPager->state>=SHARED_LOCK ); /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. */ - if( !isErrorReset ){ - assert( pPager->state <= PAGER_SHARED ); - rc = hasHotJournal(pPager, &isHotJournal); - if( rc!=SQLITE_OK ){ - goto failed; - } - } - if( isErrorReset || isHotJournal ){ + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); + } + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ /* Get an EXCLUSIVE lock on the database file. At this point it is ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the ** database file, detect the RESERVED lock, and conclude that the ** database is safe to read while this process is still rolling the @@ -37785,62 +39371,49 @@ ** ** Because the intermediate RESERVED lock is not requested, any ** other process attempting to access the database file will get to ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. - */ - if( pPager->statefd, EXCLUSIVE_LOCK); - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } - pPager->state = PAGER_EXCLUSIVE; - } - - /* Open the journal for read/write access. This is because in - ** exclusive-access mode the file descriptor will be kept open and - ** possibly used for a transaction later on. On some systems, the - ** OsTruncate() call used in exclusive-access mode also requires - ** a read/write file handle. - */ - if( !isOpen(pPager->jfd) ){ - int res; - rc = sqlite3OsAccess(pVfs,pPager->zJournal,SQLITE_ACCESS_EXISTS,&res); - if( rc==SQLITE_OK ){ - if( res ){ - int fout = 0; - int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; - assert( !pPager->tempFile ); - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ - rc = SQLITE_CANTOPEN_BKPT; - sqlite3OsClose(pPager->jfd); - } - }else{ - /* If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it - ** may mean that the pager was in the error-state when this - ** function was called and the journal file does not exist. */ - rc = pager_end_transaction(pPager, 0); - } - } - } - if( rc!=SQLITE_OK ){ - goto failed; - } - - /* Reset the journal status fields to indicates that we have no - ** rollback journal at this time. */ - pPager->journalStarted = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - - /* Make sure the journal file has been synced to disk. */ + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + goto failed; + } + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before ** playing back the hot-journal so that we don't end up with ** an inconsistent cache. Sync the hot journal before playing @@ -37847,25 +39420,50 @@ ** it back since the process that crashed and left the hot journal ** probably did not sync it and we are required to always sync ** the journal before playing it back. */ if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); rc = pagerSyncHotJournal(pPager); if( rc==SQLITE_OK ){ rc = pager_playback(pPager, 1); - } - if( rc!=SQLITE_OK ){ - rc = pager_error(pPager, rc); - goto failed; - } - } - assert( (pPager->state==PAGER_SHARED) - || (pPager->exclusiveMode && pPager->state>PAGER_SHARED) + pPager->eState = PAGER_OPEN; + } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) ); } - if( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 ){ + if( !pPager->tempFile + && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) + ){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database ** has been modified. If the database has changed, flush the ** cache. @@ -37878,18 +39476,15 @@ ** ** There is a vanishingly small chance that a change will not be ** detected. The chance of an undetected change is so small that ** it can be neglected. */ - int nPage = 0; + Pgno nPage = 0; char dbFileVers[sizeof(pPager->dbFileVers)]; - sqlite3PagerPagecount(pPager, &nPage); - if( pPager->errCode ){ - rc = pPager->errCode; - goto failed; - } + rc = pagerPagecount(pPager, &nPage); + if( rc ) goto failed; if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ @@ -37901,22 +39496,36 @@ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); } } - assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); /* If there is a WAL file in the file-system, open this database in WAL ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); } failed: if( rc!=SQLITE_OK ){ - /* pager_unlock() is a no-op for exclusive mode and in-memory databases. */ + assert( !MEMDB ); pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; } return rc; } /* @@ -37926,13 +39535,11 @@ ** Except, in locking_mode=EXCLUSIVE when there is nothing to in ** the rollback journal, the unlock is not performed and there is ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) - && (!pPager->exclusiveMode || pPager->journalOff>0) - ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } /* @@ -37992,20 +39599,20 @@ int noContent /* Do not bother reading content from disk if true */ ){ int rc; PgHdr *pPg; + assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( pPager->state>PAGER_UNLOCK ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; } /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK && pPager->errCode!=SQLITE_FULL ){ + if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -38027,11 +39634,10 @@ return SQLITE_OK; }else{ /* The pager cache has created a new page. Its content needs to ** be initialized. */ - int nMax; PAGER_INCR(pPager->nMiss); pPg = *ppPage; pPg->pPager = pPager; @@ -38040,16 +39646,11 @@ if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ rc = SQLITE_CORRUPT_BKPT; goto pager_acquire_err; } - rc = sqlite3PagerPagecount(pPager, &nMax); - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - - if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ + if( MEMDB || pPager->dbSizefd) ){ if( pgno>pPager->mxPgno ){ rc = SQLITE_FULL; goto pager_acquire_err; } if( noContent ){ @@ -38075,13 +39676,11 @@ rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } } -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } return SQLITE_OK; pager_acquire_err: @@ -38096,13 +39695,11 @@ } /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. Also, return 0 if the -** pager is in PAGER_UNLOCK state when this function is called, -** or if the pager is in an error state other than SQLITE_FULL. +** or 0 if the page is not in cache. ** ** See also sqlite3PagerGet(). The difference between this routine ** and sqlite3PagerGet() is that _get() will go to the disk and read ** in the page if the page is not already in cache. This routine ** returns NULL if the page is not in cache or if a disk I/O error @@ -38111,11 +39708,11 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ PgHdr *pPg = 0; assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->state > PAGER_UNLOCK ); + assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); return pPg; } /* @@ -38156,73 +39753,71 @@ ** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or ** an IO error code if opening or writing the journal file fails. */ static int pager_open_journal(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ - int nPage; /* Size of database file */ sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ - assert( pPager->state>=PAGER_RESERVED ); - assert( pPager->useJournal ); - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF ); + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); /* If already in the error state, this function is a no-op. But on ** the other hand, this routine is never called if we are already in ** an error state. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - testcase( pPager->dbSizeValid==0 ); - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; - pPager->pInJournal = sqlite3BitvecCreate(nPage); - if( pPager->pInJournal==0 ){ - return SQLITE_NOMEM; - } - - /* Open the journal file if it is not already open. */ - if( !isOpen(pPager->jfd) ){ - if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ - sqlite3MemJournalOpen(pPager->jfd); - }else{ - const int flags = /* VFS flags to open journal file */ - SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - (pPager->tempFile ? - (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): - (SQLITE_OPEN_MAIN_JOURNAL) - ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - rc = sqlite3JournalOpen( - pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) - ); -#else - rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); -#endif - } - assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); - } - - - /* Write the first journal header to the journal file and open - ** the sub-journal if necessary. - */ - if( rc==SQLITE_OK ){ - /* TODO: Check if all of these are really required. */ - pPager->dbOrigSize = pPager->dbSize; - pPager->journalStarted = 0; - pPager->needSync = 0; - pPager->nRec = 0; - pPager->journalOff = 0; - pPager->setMaster = 0; - pPager->journalHdr = 0; - rc = writeJournalHdr(pPager); + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + const int flags = /* VFS flags to open journal file */ + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + (pPager->tempFile ? + (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL): + (SQLITE_OPEN_MAIN_JOURNAL) + ); + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + rc = sqlite3JournalOpen( + pVfs, pPager->zJournal, pPager->jfd, flags, jrnlBufferSize(pPager) + ); + #else + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, flags, 0); + #endif + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setMaster = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); + } } if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; } + return rc; } /* ** Begin a write-transaction on the specified pager object. If a @@ -38231,18 +39826,10 @@ ** If the exFlag argument is false, then acquire at least a RESERVED ** lock on the database file. If exFlag is true, then acquire at least ** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** -** If this is not a temporary or in-memory file and, the journal file is -** opened if it has not been already. For a temporary file, the opening -** of the journal file is deferred until there is an actual need to -** write to the journal. TODO: Why handle temporary files differently? -** -** If the journal file is opened (or if it is already open), then a -** journal-header is written to the start of it. -** ** If the subjInMemory argument is non-zero, then any sub-journal opened ** within this transaction will be opened as an in-memory file. This ** has no effect if the sub-journal is already opened (as it may be when ** running in exclusive mode) or if the transaction does not require a ** sub-journal. If the subjInMemory argument is zero, then any required @@ -38249,24 +39836,24 @@ ** sub-journal is implemented in-memory if pPager is an in-memory database, ** or using a temporary file otherwise. */ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ int rc = SQLITE_OK; - assert( pPager->state!=PAGER_UNLOCK ); + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - if( pPager->state==PAGER_SHARED ){ + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); - assert( !MEMDB && !pPager->tempFile ); if( pagerUseWal(pPager) ){ /* If the pager is configured to use locking_mode=exclusive, and an ** exclusive lock on the database is not already held, obtain it now. */ if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ - rc = sqlite3OsLock(pPager->fd, EXCLUSIVE_LOCK); - pPager->state = PAGER_SHARED; + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ return rc; } sqlite3WalExclusiveMode(pPager->pWal, 1); } @@ -38273,56 +39860,44 @@ /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_RESERVED state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. - ** - ** WAL mode sets Pager.state to PAGER_RESERVED when it has an open - ** transaction, but never to PAGER_EXCLUSIVE. This is because in - ** PAGER_EXCLUSIVE state the code to roll back savepoint transactions - ** may copy data from the sub-journal into the database file as well - ** as into the page cache. Which would be incorrect in WAL mode. */ rc = sqlite3WalBeginWriteTransaction(pPager->pWal); - if( rc==SQLITE_OK ){ - pPager->dbOrigSize = pPager->dbSize; - pPager->state = PAGER_RESERVED; - pPager->journalOff = 0; - } - - assert( rc!=SQLITE_OK || pPager->state==PAGER_RESERVED ); - assert( rc==SQLITE_OK || pPager->state==PAGER_SHARED ); }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter ** is true, then immediately upgrade this to an EXCLUSIVE lock. The ** busy-handler callback can be used when upgrading to the EXCLUSIVE ** lock, but not when obtaining the RESERVED lock. */ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } - } - } - - /* No need to open the journal file at this time. It will be - ** opened before it is written to. If we defer opening the journal, - ** we might save the work of creating a file if the transaction - ** ends up being a no-op. - */ - - if( rc!=SQLITE_OK ){ - assert( !pPager->dbModified ); - /* Ignore any IO error that occurs within pager_end_transaction(). The - ** purpose of this call is to reset the internal state of the pager - ** sub-system. It doesn't matter if the journal-file is not properly - ** finalized at this point (since it is not a valid journal file anyway). - */ - pager_end_transaction(pPager, 0); - } + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); } PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); return rc; } @@ -38337,110 +39912,98 @@ static int pager_write(PgHdr *pPg){ void *pData = pPg->pData; Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a transaction has already been - ** started. + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. */ - assert( pPager->state>=PAGER_RESERVED ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* If an error has been previously detected, report the same error - ** again. - */ + ** again. This should not happen, but the check provides robustness. */ if( NEVER(pPager->errCode) ) return pPager->errCode; /* Higher-level routines never call this function if database is not ** writable. But check anyway, just for robustness. */ if( NEVER(pPager->readOnly) ) return SQLITE_PERM; - assert( !pPager->setMaster ); - CHECK_PAGE(pPg); + + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); /* Mark the page as dirty. If the page has already been written ** to the journal then we can return right away. */ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ assert( !pagerUseWal(pPager) ); - pPager->dbModified = 1; }else{ - - /* If we get this far, it means that the page needs to be - ** written to the transaction journal or the ckeckpoint journal - ** or both. - ** - ** Higher level routines should have already started a transaction, - ** which means they have acquired the necessary locks but the rollback - ** journal might not yet be open. - */ - assert( pPager->state>=RESERVED_LOCK ); - if( pPager->pInJournal==0 - && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && !pagerUseWal(pPager) - ){ - assert( pPager->useJournal ); - rc = pager_open_journal(pPager); - if( rc!=SQLITE_OK ) return rc; - } - pPager->dbModified = 1; /* The transaction journal now exists and we have a RESERVED or an ** EXCLUSIVE lock on the main database file. Write the current page to ** the transaction journal if it is not there already. */ - if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ - assert( !pagerUseWal(pPager) ); - if( pPg->pgno<=pPager->dbOrigSize ){ + if( !pageInJournal(pPg) && !pagerUseWal(pPager) ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize && isOpen(pPager->jfd) ){ u32 cksum; char *pData2; + i64 iOff = pPager->journalOff; /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); - assert( pPager->journalHdr <= pPager->journalOff ); + assert( pPager->journalHdr<=pPager->journalOff ); CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); cksum = pager_cksum(pPager, (u8*)pData2); - rc = write32bits(pPager->jfd, pPager->journalOff, pPg->pgno); - if( rc==SQLITE_OK ){ - rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, - pPager->journalOff + 4); - pPager->journalOff += pPager->pageSize+4; - } - if( rc==SQLITE_OK ){ - rc = write32bits(pPager->jfd, pPager->journalOff, cksum); - pPager->journalOff += 4; - } + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); - /* Even if an IO or diskfull error occurred while journalling the - ** page in the block above, set the need-sync flag for the page. - ** Otherwise, when the transaction is rolled back, the logic in - ** playback_one_page() will think that the page needs to be restored - ** in the database file. And if an IO error occurs while doing so, - ** then corruption may follow. - */ - if( !pPager->noSync ){ - pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; - } - - /* An error has occurred writing to the journal file. The - ** transaction will be rolled back by the layer above. - */ - if( rc!=SQLITE_OK ){ - return rc; - } - + pPager->journalOff += 8 + pPager->pageSize; pPager->nRec++; assert( pPager->pInJournal!=0 ); rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); testcase( rc==SQLITE_NOMEM ); assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); @@ -38448,13 +40011,12 @@ if( rc!=SQLITE_OK ){ assert( rc==SQLITE_NOMEM ); return rc; } }else{ - if( !pPager->journalStarted && !pPager->noSync ){ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ pPg->flags |= PGHDR_NEED_SYNC; - pPager->needSync = 1; } PAGERTRACE(("APPEND %d page %d needSync=%d\n", PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); } @@ -38470,11 +40032,10 @@ } } /* Update the database size and return. */ - assert( pPager->state>=PAGER_SHARED ); if( pPager->dbSizepgno ){ pPager->dbSize = pPg->pgno; } return rc; } @@ -38498,10 +40059,14 @@ PgHdr *pPg = pDbPage; Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPager) ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ int nPage = 0; /* Number of pages starting at pg1 to journal */ int ii; /* Loop counter */ @@ -38519,23 +40084,21 @@ ** an integer power of 2. It sets variable pg1 to the identifier ** of the first page of the sector pPg is located on. */ pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - rc = sqlite3PagerPagecount(pPager, (int *)&nPageCount); - if( rc==SQLITE_OK ){ - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); - } + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ @@ -38543,11 +40106,10 @@ rc = sqlite3PagerGet(pPager, pg, &pPage); if( rc==SQLITE_OK ){ rc = pager_write(pPage); if( pPage->flags&PGHDR_NEED_SYNC ){ needSync = 1; - assert(pPager->needSync); } sqlite3PagerUnref(pPage); } } }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ @@ -38563,19 +40125,18 @@ ** writing to any of these nPage pages may damage the others, the ** journal file must contain sync()ed copies of all of them ** before any of them can be written out to the database file. */ if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB && pPager->noSync==0 ); + assert( !MEMDB ); for(ii=0; iiflags |= PGHDR_NEED_SYNC; sqlite3PagerUnref(pPage); } } - assert(pPager->needSync); } assert( pPager->doNotSyncSpill==1 ); pPager->doNotSyncSpill--; }else{ @@ -38613,13 +40174,11 @@ Pager *pPager = pPg->pPager; if( (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) pPg->flags |= PGHDR_DONT_WRITE; -#ifdef SQLITE_CHECK_PAGES - pPg->pageHash = pager_pagehash(pPg); -#endif + pager_set_pagehash(pPg); } } /* ** This routine is called to increment the value of the database file @@ -38637,10 +40196,15 @@ ** by writing an updated version of page 1 using a call to the ** sqlite3OsWrite() function. */ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ int rc = SQLITE_OK; + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* Declare and initialize constant integer 'isDirect'. If the ** atomic-write optimization is enabled in this build, then isDirect ** is initialized to the value passed as the isDirectMode parameter ** to this function. Otherwise, it is always set to zero. @@ -38656,11 +40220,10 @@ UNUSED_PARAMETER(isDirectMode); #else # define DIRECT_MODE isDirectMode #endif - assert( pPager->state>=PAGER_RESERVED ); if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ u32 change_counter; /* Initial value of change-counter field */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -38711,23 +40274,47 @@ } return rc; } /* -** Sync the pager file to disk. This is a no-op for in-memory files +** Sync the database file to disk. This is a no-op for in-memory databases ** or pages with the Pager.noSync flag set. ** -** If successful, or called on a pager for which it is a no-op, this +** If successful, or if called on a pager for which it is a no-op, this ** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager){ int rc; /* Return code */ assert( !MEMDB ); if( pPager->noSync ){ rc = SQLITE_OK; }else{ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); + } + return rc; +} + +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); } return rc; } /* @@ -38761,31 +40348,38 @@ const char *zMaster, /* If not NULL, the master journal name */ int noSync /* True to omit the xSync on the db file */ ){ int rc = SQLITE_OK; /* Return code */ - /* The dbOrigSize is never set if journal_mode=OFF */ - assert( pPager->journalMode!=PAGER_JOURNALMODE_OFF || pPager->dbOrigSize==0 ); + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); /* If a prior error occurred, report that error again. */ - if( pPager->errCode ) return pPager->errCode; + if( NEVER(pPager->errCode) ) return pPager->errCode; PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", pPager->zFilename, zMaster, pPager->dbSize)); - if( MEMDB && pPager->dbModified ){ + /* If no database changes have been made, return early. */ + if( pPager->eStatepBackup); - }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ + }else{ if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, - (pPager->fullSync ? pPager->sync_flags : 0) + (pPager->fullSync ? pPager->syncFlags : 0) ); } if( rc==SQLITE_OK ){ sqlite3PcacheCleanAll(pPager->pPCache); } @@ -38819,11 +40413,11 @@ || pPager->journalMode==PAGER_JOURNALMODE_OFF || pPager->journalMode==PAGER_JOURNALMODE_WAL ); if( !zMaster && isOpen(pPager->jfd) && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbFileSize + && pPager->dbSize>=pPager->dbOrigSize && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) ){ /* Update the db file change counter via the direct-write method. The ** following call will modify the in-memory representation of page 1 ** to include the updated change counter and then write page 1 @@ -38849,17 +40443,14 @@ ** Before reading the pages with page numbers larger than the ** current value of Pager.dbSize, set dbSize back to the value ** that it took at the start of the transaction. Otherwise, the ** calls to sqlite3PagerGet() return zeroed pages instead of ** reading data from the database file. - ** - ** When journal_mode==OFF the dbOrigSize is always zero, so this - ** block never runs if journal_mode=OFF. */ #ifndef SQLITE_OMIT_AUTOVACUUM if( pPager->dbSizedbOrigSize - && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) + && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ Pgno i; /* Iterator variable */ const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ const Pgno dbSize = pPager->dbSize; /* Database image size */ pPager->dbSize = pPager->dbOrigSize; @@ -38882,18 +40473,24 @@ ** or if zMaster is NULL (no master journal), then this call is a no-op. */ rc = writeMasterJournal(pPager, zMaster); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Sync the journal file. If the atomic-update optimization is being - ** used, this call will not create the journal file or perform any - ** real IO. + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. */ - rc = syncJournal(pPager); + rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - /* Write all dirty pages to the database file. */ rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; } @@ -38902,26 +40499,27 @@ /* If the file on disk is not the same size as the database image, ** then use pager_truncate to grow or shrink the file here. */ if( pPager->dbSize!=pPager->dbFileSize ){ Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->state>=PAGER_EXCLUSIVE ); + assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } /* Finally, sync the database file. */ if( !pPager->noSync && !noSync ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } IOTRACE(("DBSYNC %p\n", pPager)) } - - pPager->state = PAGER_SYNCED; } commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } return rc; } /* @@ -38945,16 +40543,15 @@ /* This routine should not be called if a prior error has occurred. ** But if (due to a coding error elsewhere in the system) it does get ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; - /* This function should not be called if the pager is not in at least - ** PAGER_RESERVED state. **FIXME**: Make it so that this test always - ** fails - make it so that we never reach this point if we do not hold - ** all necessary locks. - */ - if( NEVER(pPager->stateeState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); /* An optimization. If the database was not actually modified during ** this transaction, the pager is running in exclusive-mode and is ** using persistent journals, then this function is a no-op. ** @@ -38963,106 +40560,80 @@ ** a hot-journal during hot-journal rollback, 0 changes will be made ** to the database file. So there is no need to zero the journal ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->dbModified==0 && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; return SQLITE_OK; } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - assert( pPager->state==PAGER_SYNCED || MEMDB || !pPager->dbModified ); rc = pager_end_transaction(pPager, pPager->setMaster); return pager_error(pPager, rc); } /* -** Rollback all changes. The database falls back to PAGER_SHARED mode. +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. ** -** This function performs two tasks: +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: ** ** 1) It rolls back the journal file, restoring all database file and ** in-memory cache pages to the state they were in when the transaction ** was opened, and +** ** 2) It finalizes the journal file, so that it is not used for hot ** rollback at any point in the future. ** -** subject to the following qualifications: -** -** * If the journal file is not yet open when this function is called, -** then only (2) is performed. In this case there is no journal file -** to roll back. -** -** * If in an error state other than SQLITE_FULL, then task (1) is -** performed. If successful, task (2). Regardless of the outcome -** of either, the error state error code is returned to the caller -** (i.e. either SQLITE_IOERR or SQLITE_CORRUPT). -** -** * If the pager is in PAGER_RESERVED state, then attempt (1). Whether -** or not (1) is successful, also attempt (2). If successful, return -** SQLITE_OK. Otherwise, enter the error state and return the first -** error code encountered. -** -** In this case there is no chance that the database was written to. -** So is safe to finalize the journal file even if the playback -** (operation 1) failed. However the pager must enter the error state -** as the contents of the in-memory cache are now suspect. -** -** * Finally, if in PAGER_EXCLUSIVE state, then attempt (1). Only -** attempt (2) if (1) is successful. Return SQLITE_OK if successful, -** otherwise enter the error state and return the error code from the -** failing operation. -** -** In this case the database file may have been written to. So if the -** playback operation did not succeed it would not be safe to finalize -** the journal file. It needs to be left in the file-system so that -** some other process can use it to restore the database state (by -** hot-journal rollback). +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. +** +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. */ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + if( pagerUseWal(pPager) ){ int rc2; - rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; - rc = pager_error(pPager, rc); - }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){ - rc = pager_end_transaction(pPager, pPager->setMaster); - }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ - if( pPager->state>=PAGER_EXCLUSIVE ){ - pager_playback(pPager, 0); - } - rc = pPager->errCode; - }else{ - if( pPager->state==PAGER_RESERVED ){ - int rc2; - rc = pager_playback(pPager, 0); - rc2 = pager_end_transaction(pPager, pPager->setMaster); - if( rc==SQLITE_OK ){ - rc = rc2; - } - }else{ - rc = pager_playback(pPager, 0); - } - - if( !MEMDB ){ - pPager->dbSizeValid = 0; - } - - /* If an error occurs during a ROLLBACK, we can no longer trust the pager - ** cache. So call pager_error() on the way out to make any error - ** persistent. - */ - rc = pager_error(pPager, rc); - } - return rc; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_end_transaction(pPager, 0); + }else{ + rc = pager_playback(pPager, 0); + } + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || (rc&0xFF)==SQLITE_IOERR ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); } /* ** Return TRUE if the database file is opened read-only. Return FALSE ** if the database is (in theory) writable. @@ -39081,13 +40652,15 @@ /* ** Return the approximate number of bytes of memory currently ** used by the pager and its associated cache. */ SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ - int perPageSize = pPager->pageSize + pPager->nExtra + 20; + int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) + + 5*sizeof(void*); return perPageSize*sqlite3PcachePagecount(pPager->pPCache) - + sqlite3MallocSize(pPager); + + sqlite3MallocSize(pPager) + + pPager->pageSize; } /* ** Return the number of references to the specified page. */ @@ -39102,12 +40675,12 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ static int a[11]; a[0] = sqlite3PcacheRefCount(pPager->pPCache); a[1] = sqlite3PcachePagecount(pPager->pPCache); a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); - a[3] = pPager->dbSizeValid ? (int) pPager->dbSize : -1; - a[4] = pPager->state; + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; a[5] = pPager->errCode; a[6] = pPager->nHit; a[7] = pPager->nMiss; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; @@ -39134,18 +40707,17 @@ ** returned. Otherwise, SQLITE_OK. */ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ int rc = SQLITE_OK; /* Return code */ int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); if( nSavepoint>nCurrent && pPager->useJournal ){ int ii; /* Iterator variable */ PagerSavepoint *aNew; /* New Pager.aSavepoint array */ - int nPage; /* Size of database file */ - - rc = sqlite3PagerPagecount(pPager, &nPage); - if( rc ) return rc; /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM ** if the allocation fails. Otherwise, zero the new portion in case a ** malloc failure occurs while populating it in the for(...) loop below. */ @@ -39158,18 +40730,18 @@ memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); pPager->aSavepoint = aNew; /* Populate the PagerSavepoint structures just allocated. */ for(ii=nCurrent; iidbSize; if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ aNew[ii].iOffset = pPager->journalOff; }else{ aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); } aNew[ii].iSubRec = pPager->nSubRec; - aNew[ii].pInSavepoint = sqlite3BitvecCreate(nPage); + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } if( pagerUseWal(pPager) ){ sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); @@ -39212,16 +40784,16 @@ ** This function may return SQLITE_NOMEM if a memory allocation fails, ** or an IO error code if an IO error occurs while rolling back a ** savepoint. If no errors occur, SQLITE_OK is returned. */ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = SQLITE_OK; + int rc = pPager->errCode; /* Return code */ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); - if( iSavepointnSavepoint ){ + if( rc==SQLITE_OK && iSavepointnSavepoint ){ int ii; /* Iterator variable */ int nNew; /* Number of remaining savepoints after this op. */ /* Figure out how many savepoints will still be active after this ** operation. Store this value in nNew. Then free resources associated @@ -39253,12 +40825,12 @@ else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } - } + return rc; } /* ** Return the full pathname of the database file. @@ -39352,10 +40924,14 @@ Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */ int rc; /* Return code */ Pgno origPgno; /* The original page number */ assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); /* In order to be able to rollback, an in-memory database must journal ** the page we are moving from. */ if( MEMDB ){ @@ -39401,15 +40977,14 @@ */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ needSyncPgno = pPg->pgno; assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); assert( pPg->flags&PGHDR_DIRTY ); - assert( pPager->needSync ); } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PgHdr.needSync was set for + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = pager_lookup(pPager, pgno); @@ -39417,67 +40992,59 @@ if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( MEMDB ){ /* Do not discard pages from an in-memory database since we might ** need to rollback later. Just move the page out of the way. */ - assert( pPager->dbSizeValid ); sqlite3PcacheMove(pPgOld, pPager->dbSize+1); }else{ sqlite3PcacheDrop(pPgOld); } } origPgno = pPg->pgno; sqlite3PcacheMove(pPg, pgno); sqlite3PcacheMakeDirty(pPg); - pPager->dbModified = 1; + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( MEMDB ){ + assert( pPgOld ); + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnref(pPgOld); + } if( needSyncPgno ){ /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by - ** loading the page into the pager-cache and setting the PgHdr.needSync + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. ** ** If the attempt to load the page into the page-cache fails, (due ** to a malloc() or IO failure), clear the bit in the pInJournal[] ** array. Otherwise, if the page is loaded and written again in ** this transaction, it may be written to the database file before ** it is synced into the journal file. This way, it may end up in ** the journal file twice, but that is not a problem. - ** - ** The sqlite3PagerGet() call may cause the journal to sync. So make - ** sure the Pager.needSync flag is set too. */ PgHdr *pPgHdr; - assert( pPager->needSync ); rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr); if( rc!=SQLITE_OK ){ if( needSyncPgno<=pPager->dbOrigSize ){ assert( pPager->pTmpSpace!=0 ); sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); } return rc; } - pPager->needSync = 1; - assert( pPager->noSync==0 && !MEMDB ); pPgHdr->flags |= PGHDR_NEED_SYNC; sqlite3PcacheMakeDirty(pPgHdr); sqlite3PagerUnref(pPgHdr); } - /* - ** For an in-memory database, make sure the original page continues - ** to exist, in case the transaction needs to roll back. Use pPgOld - ** as the original page since it has already been allocated. - */ - if( MEMDB ){ - sqlite3PcacheMove(pPgOld, origPgno); - sqlite3PagerUnref(pPgOld); - } - return SQLITE_OK; } #endif /* @@ -39510,11 +41077,12 @@ assert( eMode==PAGER_LOCKINGMODE_QUERY || eMode==PAGER_LOCKINGMODE_NORMAL || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_QUERY<0 ); assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); - if( eMode>=0 && !pPager->tempFile ){ + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ pPager->exclusiveMode = (u8)eMode; } return (int)pPager->exclusiveMode; } @@ -39538,10 +41106,17 @@ ** ** The returned indicate the current (possibly updated) journal-mode. */ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ + +#ifdef SQLITE_DEBUG + /* The print_pager_state() routine is intended to be used by the debugger + ** only. We invoke it once here to suppress a compiler warning. */ + print_pager_state(pPager); +#endif + /* The eMode parameter is always valid */ assert( eMode==PAGER_JOURNALMODE_DELETE || eMode==PAGER_JOURNALMODE_TRUNCATE || eMode==PAGER_JOURNALMODE_PERSIST @@ -39564,24 +41139,17 @@ eMode = eOld; } } if( eMode!=eOld ){ - /* When changing between rollback modes, close the journal file prior - ** to the change. But when changing from a rollback mode to WAL, keep - ** the journal open since there is a rollback-style transaction in play - ** used to convert the version numbers in the btree header. - */ - if( isOpen(pPager->jfd) && eMode!=PAGER_JOURNALMODE_WAL ){ - sqlite3OsClose(pPager->jfd); - } /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; /* When transistioning from TRUNCATE or PERSIST to any other journal - ** mode except WAL (and we are not in locking_mode=EXCLUSIVE) then + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); @@ -39598,28 +41166,34 @@ ** ** Before deleting the journal file, obtain a RESERVED lock on the ** database file. This ensures that the journal file is not deleted ** while it is in use by some other client. */ - int rc = SQLITE_OK; - int state = pPager->state; - if( statestate==PAGER_SHARED ){ - assert( rc==SQLITE_OK ); - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - } - if( rc==SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); } - if( rc==SQLITE_OK && state==PAGER_SHARED ){ - sqlite3OsUnlock(pPager->fd, SHARED_LOCK); - }else if( state==PAGER_UNLOCK ){ - pager_unlock(pPager); - } - assert( state==pPager->state ); } } /* Return the new journal mode */ return (int)pPager->journalMode; @@ -39636,11 +41210,12 @@ ** Return TRUE if the pager is in a state where it is OK to change the ** journalmode. Journalmode changes can only happen when the database ** is unmodified. */ SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ - if( pPager->dbModified ) return 0; + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; return 1; } /* @@ -39672,14 +41247,12 @@ */ SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pWal ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3WalCheckpoint(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, zBuf - ); + rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, zBuf); } return rc; } SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ @@ -39690,50 +41263,105 @@ ** Return true if the underlying VFS for the given pager supports the ** primitives necessary for write-ahead logging. */ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ const sqlite3_io_methods *pMethods = pPager->fd->pMethods; - return pMethods->iVersion>=2 && pMethods->xShmOpen!=0; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); } + +/* +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. +*/ +static int pagerExclusiveLock(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + /* If the attempt to grab the pending lock failed, release the + ** exclusive lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + return rc; +} + +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK || pPager->noReadlock); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); + } + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, &pPager->pWal + ); + } + + return rc; +} + /* ** The caller must be holding a SHARED lock on the database file to call ** this function. ** ** If the pager passed as the first argument is open on a real database ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, ** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pisOpen is +** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** ** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pisOpen to 1 and return SQLITE_OK +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK ** without doing anything. */ SQLITE_PRIVATE int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ - int *pisOpen /* OUT: Set to true if call is a no-op */ + int *pbOpen /* OUT: Set to true if call is a no-op */ ){ int rc = SQLITE_OK; /* Return code */ - assert( pPager->state>=PAGER_SHARED ); - assert( (pisOpen==0 && !pPager->tempFile && !pPager->pWal) || *pisOpen==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; - /* Open the connection to the log file. If this operation fails, - ** (e.g. due to malloc() failure), unlock the database file and - ** return an error code. - */ - rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, &pPager->pWal); + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + + rc = pagerOpenWal(pPager); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; } }else{ - *pisOpen = 1; + *pbOpen = 1; } return rc; } @@ -39755,37 +41383,30 @@ ** it may need to be checkpointed before the connection can switch to ** rollback mode. Open it now so this can happen. */ if( !pPager->pWal ){ int logexists = 0; - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); + rc = pagerLockDb(pPager, SHARED_LOCK); if( rc==SQLITE_OK ){ rc = sqlite3OsAccess( pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists ); } if( rc==SQLITE_OK && logexists ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, - pPager->zWal, &pPager->pWal); + rc = pagerOpenWal(pPager); } } /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ - rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); + rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, - (pPager->noSync ? 0 : pPager->sync_flags), - pPager->pageSize, (u8*)pPager->pTmpSpace - ); + rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; - }else{ - /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock - ** that we did get back to SHARED. */ - sqlite3OsUnlock(pPager->fd, SQLITE_LOCK_SHARED); } } return rc; } @@ -40101,18 +41722,22 @@ /* ** The following object holds a copy of the wal-index header content. ** ** The actual header in the wal-index consists of two copies of this ** object. +** +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. */ struct WalIndexHdr { u32 iVersion; /* Wal-index version */ u32 unused; /* Unused (padding) field */ u32 iChange; /* Counter incremented each transaction */ u8 isInit; /* 1 when initialized */ u8 bigEndCksum; /* True if checksums in WAL are big-endian */ - u16 szPage; /* Database page size in bytes */ + u16 szPage; /* Database page size in bytes. 1==64K */ u32 mxFrame; /* Index of last valid frame in the WAL */ u32 nPage; /* Size of database in pages */ u32 aFrameCksum[2]; /* Checksum of last frame in log */ u32 aSalt[2]; /* Two salt values copied from WAL header */ u32 aCksum[2]; /* Checksum over all prior fields */ @@ -40219,24 +41844,31 @@ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ int nWiData; /* Size of array apWiData */ volatile u32 **apWiData; /* Pointer to wal-index content in memory */ - u16 szPage; /* Database page size */ + u32 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ - u8 isWIndexOpen; /* True if ShmOpen() called on pDbFd */ u8 writeLock; /* True if in a write transaction */ u8 ckptLock; /* True if holding a checkpoint lock */ + u8 readOnly; /* True if the WAL file is open read-only */ WalIndexHdr hdr; /* Wal-index header for current transaction */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif }; +/* +** Candidate values for Wal.exclusiveMode. +*/ +#define WAL_NORMAL_MODE 0 +#define WAL_EXCLUSIVE_MODE 1 +#define WAL_HEAPMEMORY_MODE 2 + /* ** Each page of the wal-index mapping contains a hash-table made up of ** an array of HASHTABLE_NSLOT elements of the following type. */ typedef u16 ht_slot; @@ -40319,13 +41951,18 @@ pWal->nWiData = iPage+1; } /* Request a pointer to the required page from the VFS */ if( pWal->apWiData[iPage]==0 ){ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + } } *ppPage = pWal->apWiData[iPage]; assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; @@ -40403,10 +42040,16 @@ } aOut[0] = s1; aOut[1] = s2; } + +static void walShmBarrier(Wal *pWal){ + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); + } +} /* ** Write the header information in pWal->hdr into the wal-index. ** ** The checksum on pWal->hdr is updated before it is written. @@ -40418,11 +42061,11 @@ assert( pWal->writeLock ); pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); } /* ** This function encodes a single frame header and writes it to a buffer @@ -40890,11 +42533,11 @@ || szPage<512 ){ goto finished; } pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); /* Verify that the WAL header checksum is correct */ walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, @@ -40940,11 +42583,13 @@ /* If nTruncate is non-zero, this is a commit record. */ if( nTruncate ){ pWal->hdr.mxFrame = iFrame; pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; } } @@ -40965,10 +42610,21 @@ */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; pInfo->aReadMark[0] = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; + + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + pWal->hdr.nPage, pWal->zWalName + ); + } } recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); walUnlockExclusive(pWal, iLock, nLock); @@ -40977,13 +42633,18 @@ /* ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->isWIndexOpen ){ - sqlite3OsShmClose(pWal->pDbFd, isDelete); - pWal->isWIndexOpen = 0; + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void *)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + }else{ + sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } } /* ** Open a connection to the WAL file zWalName. The database file must @@ -41002,10 +42663,11 @@ */ SQLITE_PRIVATE int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ + int bNoShm, /* True to run in heap-memory mode */ Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc; /* Return Code */ Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ @@ -41035,17 +42697,17 @@ pRet->pVfs = pVfs; pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->zWalName = zWalName; - rc = sqlite3OsShmOpen(pDbFd); + pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ - if( rc==SQLITE_OK ){ - pRet->isWIndexOpen = 1; - flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_JOURNAL); - rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + pRet->readOnly = 1; } if( rc!=SQLITE_OK ){ walIndexClose(pRet, 0); sqlite3OsClose(pRet->pWalFd); @@ -41329,18 +42991,22 @@ int sync_flags, /* Flags for OsSync() (or 0) */ int nBuf, /* Size of zBuf in bytes */ u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ - int szPage = pWal->hdr.szPage; /* Database page-size */ + int szPage; /* Database page-size */ WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Wal frame containing data for iDbpage */ u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); if( pWal->hdr.mxFrame==0 ) return SQLITE_OK; /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); if( rc!=SQLITE_OK ){ @@ -41347,11 +43013,11 @@ return rc; } assert( pIter ); /*** TODO: Move this test out to the caller. Make it an assert() here ***/ - if( pWal->hdr.szPage!=nBuf ){ + if( szPage!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; goto walcheckpoint_out; } /* Compute in mxSafeFrame the index of the last frame of the WAL that is @@ -41358,10 +43024,11 @@ ** safe to write into the database. Frames beyond mxSafeFrame might ** overwrite database pages that are in use by active readers and thus ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; pInfo = walCkptInfo(pWal); for(i=1; iaReadMark[i]; if( mxSafeFrame>=y ){ assert( y<=pWal->hdr.mxFrame ); @@ -41378,22 +43045,34 @@ } if( pInfo->nBackfillnBackfill; /* Sync the WAL to disk */ if( sync_flags ){ rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } + + /* If the database file may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } + } /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue; + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ) continue; iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; @@ -41452,11 +43131,13 @@ ** ** The EXCLUSIVE lock is not released before returning. */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - pWal->exclusiveMode = 1; + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf); if( rc==SQLITE_OK ){ isDelete = 1; } } @@ -41496,11 +43177,11 @@ WalIndexHdr volatile *aHdr; /* Header in shared memory */ /* The first page of the wal-index must be mapped at this point. */ assert( pWal->nWiData>0 && pWal->apWiData[0] ); - /* Read the header. This might happen currently with a write to the + /* Read the header. This might happen concurrently with a write to the ** same area of shared memory on a different CPU in a SMP, ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. ** ** There are two copies of the header at the beginning of the wal-index. @@ -41508,11 +43189,11 @@ ** Memory barriers are used to prevent the compiler or the hardware from ** reordering the reads and writes. */ aHdr = walIndexHdr(pWal); memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ return 1; /* Dirty read */ } @@ -41525,11 +43206,13 @@ } if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ *pChanged = 1; memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); - pWal->szPage = pWal->hdr.szPage; + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); } /* The header was successfully read. Return zero. */ return 0; } @@ -41680,12 +43363,20 @@ ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY ** would be technically correct. But the race is benign since with ** WAL_RETRY this routine will be called again and will probably be ** right on the second iteration. */ - rc = walLockShared(pWal, WAL_RECOVER_LOCK); - if( rc==SQLITE_OK ){ + if( pWal->apWiData[0]==0 ){ + /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. + ** We assume this is a transient condition, so return WAL_RETRY. The + ** xShmMap() implementation used by the default unix and win32 VFS + ** modules may return SQLITE_BUSY due to a race condition in the + ** code that determines whether or not the shared-memory region + ** must be zeroed before the requested page is returned. + */ + rc = WAL_RETRY; + }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ walUnlockShared(pWal, WAL_RECOVER_LOCK); rc = WAL_RETRY; }else if( rc==SQLITE_BUSY ){ rc = SQLITE_BUSY_RECOVERY; } @@ -41699,11 +43390,11 @@ if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. */ rc = walLockShared(pWal, WAL_READ_LOCK(0)); - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); if( rc==SQLITE_OK ){ if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, @@ -41793,11 +43484,11 @@ ** date before proceeding. That would not be possible without somehow ** blocking writers. It only guarantees that a dangerous checkpoint or ** log-wrap (either of which would require an exclusive lock on ** WAL_READ_LOCK(mxI)) has not occurred since the snapshot was valid. */ - sqlite3OsShmBarrier(pWal->pDbFd); + walShmBarrier(pWal); if( pInfo->aReadMark[mxI]!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); return WAL_RETRY; @@ -41836,10 +43527,11 @@ /* ** Finish with a read transaction. All this does is release the ** read-lock. */ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); if( pWal->readLock>=0 ){ walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); pWal->readLock = -1; } } @@ -41946,11 +43638,17 @@ /* If iRead is non-zero, then it is the log frame number that contains the ** required page. Read and return data from the log file. */ if( iRead ){ - i64 iOffset = walFrameOffset(iRead, pWal->hdr.szPage) + WAL_FRAME_HDRSIZE; + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; *pInWal = 1; /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ return sqlite3OsRead(pWal->pWalFd, pOut, nOut, iOffset); } @@ -41958,15 +43656,17 @@ return SQLITE_OK; } /* -** Set *pPgno to the size of the database file (or zero, if unknown). +** Return the size of the database in pages (or zero, if unknown). */ -SQLITE_PRIVATE void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ - assert( pWal->readLock>=0 || pWal->lockError ); - *pPgno = pWal->hdr.nPage; +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; + } + return 0; } /* ** This function starts a write transaction on the WAL. @@ -41985,10 +43685,14 @@ int rc; /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); + + if( pWal->readOnly ){ + return SQLITE_READONLY; + } /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. */ rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); @@ -42013,12 +43717,14 @@ /* ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. */ SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; + if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } return SQLITE_OK; } /* ** If any data has been written (but not committed) to the log file, this @@ -42032,11 +43738,11 @@ ** Otherwise, if the callback function does not return an error, this ** function returns SQLITE_OK. */ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - if( pWal->writeLock ){ + if( ALWAYS(pWal->writeLock) ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; /* Restore the clients cache of the wal-index header to the state it ** was in before the client began writing to the database. @@ -42120,11 +43826,11 @@ ** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left ** unchanged. ** ** SQLITE_OK is returned if no error is encountered (regardless of whether ** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned -** if some error +** if an error occurs. */ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; int cnt; @@ -42153,10 +43859,12 @@ walIndexWriteHdr(pWal); pInfo->nBackfill = 0; for(i=1; iaReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; } } walUnlockShared(pWal, WAL_READ_LOCK(0)); pWal->readLock = -1; cnt = 0; @@ -42221,11 +43929,11 @@ memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - pWal->szPage = (u16)szPage; + pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; pWal->hdr.aFrameCksum[1] = aCksum[1]; rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); @@ -42232,11 +43940,11 @@ WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); if( rc!=SQLITE_OK ){ return rc; } } - assert( pWal->szPage==szPage ); + assert( (int)pWal->szPage==szPage ); /* Write the log file. */ for(p=pList; p; p=p->pDirty){ u32 nDbsize; /* Db-size field for frame header */ i64 iOffset; /* Write offset in log file */ @@ -42316,11 +44024,13 @@ rc = walIndexAppend(pWal, iFrame, pLast->pgno); } if( rc==SQLITE_OK ){ /* Update the private copy of the header. */ - pWal->hdr.szPage = (u16)szPage; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); pWal->hdr.mxFrame = iFrame; if( isCommit ){ pWal->hdr.iChange++; pWal->hdr.nPage = nTruncate; } @@ -42417,17 +44127,18 @@ ** WAL is already in exclusive-locking mode - meaning that this ** routine is a no-op. The pager must already hold the exclusive lock ** on the main database file before invoking this operation. ** ** If op is negative, then do a dry-run of the op==1 case but do -** not actually change anything. The pager uses this to see if it +** not actually change anything. The pager uses this to see if it ** should acquire the database exclusive lock prior to invoking ** the op==1 case. */ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ int rc; assert( pWal->writeLock==0 ); + assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); /* pWal->readLock is usually set, but might be -1 if there was a ** prior error while attempting to acquire are read-lock. This cannot ** happen if the connection is actually in exclusive mode (as no xShmLock ** locks are taken in this case). Nor should the pager attempt to @@ -42456,10 +44167,19 @@ }else{ rc = pWal->exclusiveMode==0; } return rc; } + +/* +** Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ + return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); +} #endif /* #ifndef SQLITE_OMIT_WAL */ /************** End of wal.c *************************************************/ /************** Begin file btmutex.c *****************************************/ @@ -42528,11 +44248,11 @@ ** ** FORMAT DETAILS ** ** The file is divided into pages. The first page is called page 1, ** the second is page 2, and so forth. A page number of zero indicates -** "no such page". The page size can be any power of 2 between 512 and 32768. +** "no such page". The page size can be any power of 2 between 512 and 65536. ** Each page can be either a btree page, a freelist page, an overflow ** page, or a pointer-map page. ** ** The first page is always a btree page. The first 100 bytes of the first ** page contain a special header (the "file header") that describes the file. @@ -42890,22 +44610,23 @@ MemPage *pPage1; /* First page of the database */ u8 readOnly; /* True if the underlying file is readonly */ u8 pageSizeFixed; /* True if the page size can no longer be changed */ u8 secureDelete; /* True if secure_delete is enabled */ u8 initiallyEmpty; /* Database is empty at start of transaction */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ #ifndef SQLITE_OMIT_AUTOVACUUM u8 autoVacuum; /* True if auto-vacuum is enabled */ u8 incrVacuum; /* True if incr-vacuum is enabled */ #endif - u16 pageSize; /* Total number of bytes on a page */ - u16 usableSize; /* Number of usable bytes on each page */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ u8 inTransaction; /* Transaction state */ u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ int nTransaction; /* Number of open transactions (read + write) */ u32 nPage; /* Number of pages in the database */ void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ sqlite3_mutex *mutex; /* Non-recursive mutex required to access this struct */ @@ -43500,11 +45221,20 @@ # define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} #else # define TRACE(X) #endif - +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) #ifndef SQLITE_OMIT_SHARED_CACHE /* ** A list of BtShared objects that are eligible for participation ** in shared cache. This variable has file scope during normal builds, @@ -44189,15 +45919,20 @@ #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Given a page number of a regular database page, return the page ** number for the pointer-map page that contains the entry for the ** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. */ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ int nPagesPerMapPage; Pgno iPtrMap, ret; assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; nPagesPerMapPage = (pBt->usableSize/5)+1; iPtrMap = (pgno-2)/nPagesPerMapPage; ret = (iPtrMap*nPagesPerMapPage) + 2; if( ret==PENDING_BYTE_PAGE(pBt) ){ ret++; @@ -44622,21 +46357,21 @@ assert( nByte < usableSize-8 ); nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); if( gap>top ) return SQLITE_CORRUPT_BKPT; testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); if( nFrag>=60 ){ /* Always defragment highly fragmented pages */ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); }else if( gap+2<=top ){ /* Search the freelist looking for a free slot big enough to satisfy ** the request. The allocation is made from the first free slot in ** the list that is large enough to accomadate it. */ @@ -44674,11 +46409,11 @@ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ rc = defragmentPage(pPage); if( rc ) return rc; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); assert( gap+nByte<=top ); } /* Allocate memory from the gap in between the cell pointer array @@ -44840,28 +46575,28 @@ if( !pPage->isInit ){ u16 pc; /* Address of a freeblock within pPage->aData[] */ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ BtShared *pBt; /* The main btree structure */ - u16 usableSize; /* Amount of usable space on each page */ + int usableSize; /* Amount of usable space on each page */ u16 cellOffset; /* Offset from start of page to first cell pointer */ - u16 nFree; /* Number of unused bytes on the page */ - u16 top; /* First byte of the cell content area */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */ int iCellLast; /* Last possible cell or freeblock offset */ pBt = pPage->pBt; hdr = pPage->hdrOffset; data = pPage->aData; if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; - top = get2byte(&data[hdr+5]); + top = get2byteNotZero(&data[hdr+5]); pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } @@ -44956,17 +46691,17 @@ data[hdr] = (char)flags; first = hdr + 8 + 4*((flags&PTF_LEAF)==0 ?1:0); memset(&data[hdr+1], 0, 4); data[hdr+7] = 0; put2byte(&data[hdr+5], pBt->usableSize); - pPage->nFree = pBt->usableSize - first; + pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->hdrOffset = hdr; pPage->cellOffset = first; pPage->nOverflow = 0; - assert( pBt->pageSize>=512 && pBt->pageSize<=32768 ); - pPage->maskPage = pBt->pageSize - 1; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nCell = 0; pPage->isInit = 1; } @@ -45126,15 +46861,24 @@ /* ** Open a database file. ** ** zFilename is the name of the database file. If zFilename is NULL -** a new database with a random name is created. This randomly named -** database file will be deleted when sqlite3BtreeClose() is called. +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** ** If zFilename is ":memory:" then an in-memory database is created ** that is automatically destroyed when it is closed. ** +** The "flags" parameter is a bitmask that might contain bits +** BTREE_OMIT_JOURNAL and/or BTREE_NO_READLOCK. The BTREE_NO_READLOCK +** bit is also set if the SQLITE_NoReadlock flags is set in db->flags. +** These flags are passed through into sqlite3PagerOpen() and must +** be the same values as PAGER_OMIT_JOURNAL and PAGER_NO_READLOCK. +** ** If the database is already opened in the same database connection ** and we are in shared cache mode, then the open will fail with an ** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared ** objects in the same database connection since doing so will lead ** to problems with locking. @@ -45151,27 +46895,43 @@ Btree *p; /* Handle to return */ sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ int rc = SQLITE_OK; /* Result code from this function */ u8 nReserve; /* Byte of unused space on each page */ unsigned char zDbHeader[100]; /* Database header content */ + + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; /* Set the variable isMemdb to true for an in-memory database, or - ** false for a file-based database. This symbol is only required if - ** either of the shared-data or autovacuum features are compiled - ** into the library. + ** false for a file-based database. */ -#if !defined(SQLITE_OMIT_SHARED_CACHE) || !defined(SQLITE_OMIT_AUTOVACUUM) - #ifdef SQLITE_OMIT_MEMORYDB - const int isMemdb = 0; - #else - const int isMemdb = zFilename && !strcmp(zFilename, ":memory:"); - #endif +#ifdef SQLITE_OMIT_MEMORYDB + const int isMemdb = 0; +#else + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)); #endif assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( db->flags & SQLITE_NoReadlock ){ + flags |= BTREE_NO_READLOCK; + } + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } pVfs = db->pVfs; p = sqlite3MallocZero(sizeof(Btree)); if( !p ){ return SQLITE_NOMEM; } @@ -45185,11 +46945,11 @@ #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* ** If this Btree is a candidate for shared cache, try to find an ** existing BtShared object that we can share with */ - if( isMemdb==0 && zFilename && zFilename[0] ){ + if( isMemdb==0 && isTempDb==0 ){ if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ int nFullPathname = pVfs->mxPathname+1; char *zFullPathname = sqlite3Malloc(nFullPathname); sqlite3_mutex *mutexShared; p->sharable = 1; @@ -45260,10 +47020,11 @@ rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ goto btree_open_out; } + pBt->openFlags = (u8)flags; pBt->db = db; sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); p->pBt = pBt; pBt->pCursor = 0; @@ -45270,11 +47031,11 @@ pBt->pPage1 = 0; pBt->readOnly = sqlite3PagerIsreadonly(pBt->pPager); #ifdef SQLITE_SECURE_DELETE pBt->secureDelete = 1; #endif - pBt->pageSize = get2byte(&zDbHeader[16]); + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ pBt->pageSize = 0; #ifndef SQLITE_OMIT_AUTOVACUUM /* If the magic name ":memory:" will create an in-memory database, then @@ -45364,10 +47125,18 @@ sqlite3PagerClose(pBt->pPager); } sqlite3_free(pBt); sqlite3_free(p); *ppBtree = 0; + }else{ + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); + } } if( mutexOpen ){ assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } @@ -45472,11 +47241,11 @@ assert( !pBt->pCursor ); sqlite3PagerClose(pBt->pPager); if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } - sqlite3_free(pBt->pSchema); + sqlite3DbFree(0, pBt->pSchema); freeTempSpace(pBt); sqlite3_free(pBt); } #ifndef SQLITE_OMIT_SHARED_CACHE @@ -45521,15 +47290,21 @@ ** there is a high probability of damage) Level 2 is the default. There ** is a very low but non-zero probability of damage. Level 3 reduces the ** probability of damage to near zero but with a write performance reduction. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS -SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel(Btree *p, int level, int fullSync){ +SQLITE_PRIVATE int sqlite3BtreeSetSafetyLevel( + Btree *p, /* The btree to set the safety level on */ + int level, /* PRAGMA synchronous. 1=OFF, 2=NORMAL, 3=FULL */ + int fullSync, /* PRAGMA fullfsync. */ + int ckptFullSync /* PRAGMA checkpoint_fullfync */ +){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); + assert( level>=1 && level<=3 ); sqlite3BtreeEnter(p); - sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync); + sqlite3PagerSetSafetyLevel(pBt->pPager, level, fullSync, ckptFullSync); sqlite3BtreeLeave(p); return SQLITE_OK; } #endif @@ -45584,11 +47359,11 @@ assert( nReserve>=0 && nReserve<=255 ); if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pPage1 && !pBt->pCursor ); - pBt->pageSize = (u16)pageSize; + pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); pBt->usableSize = pBt->pageSize - (u16)nReserve; if( iFix ) pBt->pageSizeFixed = 1; @@ -45719,19 +47494,17 @@ /* Do some checking to help insure the file we opened really is ** a valid database file. */ nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - if( (rc = sqlite3PagerPagecount(pBt->pPager, &nPageFile))!=SQLITE_OK ){; - goto page1_init_failed; - } + sqlite3PagerPagecount(pBt->pPager, &nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; } if( nPage>0 ){ - int pageSize; - int usableSize; + u32 pageSize; + u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -45778,28 +47551,29 @@ ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } - pageSize = get2byte(&page1[16]); - if( ((pageSize-1)&pageSize)!=0 || pageSize<512 || - (SQLITE_MAX_PAGE_SIZE<32768 && pageSize>SQLITE_MAX_PAGE_SIZE) + pageSize = (page1[16]<<8) | (page1[17]<<16); + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 ){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); usableSize = pageSize - page1[20]; - if( pageSize!=pBt->pageSize ){ + if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size ** of BtShared.pageSize, we have discovered that the page-size is ** actually pageSize. Unlock the database, leave pBt->pPage1 at ** zero and return SQLITE_OK. The caller will call this function ** again with the correct page-size. */ releasePage(pPage1); - pBt->usableSize = (u16)usableSize; - pBt->pageSize = (u16)pageSize; + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); return rc; } @@ -45808,12 +47582,12 @@ goto page1_init_failed; } if( usableSize<480 ){ goto page1_init_failed; } - pBt->pageSize = (u16)pageSize; - pBt->usableSize = (u16)usableSize; + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); #endif } @@ -45825,18 +47599,18 @@ ** 2-byte pointer to the cell ** 4-byte child pointer ** 9-byte nKey value ** 4-byte nData value ** 4-byte overflow page pointer - ** So a cell consists of a 2-byte poiner, a header which is as much as + ** So a cell consists of a 2-byte pointer, a header which is as much as ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow ** page pointer. */ - pBt->maxLocal = (pBt->usableSize-12)*64/255 - 23; - pBt->minLocal = (pBt->usableSize-12)*32/255 - 23; - pBt->maxLeaf = pBt->usableSize - 35; - pBt->minLeaf = (pBt->usableSize-12)*32/255 - 23; + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; pBt->nPage = nPage; return SQLITE_OK; @@ -45885,11 +47659,12 @@ data = pP1->aData; rc = sqlite3PagerWrite(pP1->pDbPage); if( rc ) return rc; memcpy(data, zMagicHeader, sizeof(zMagicHeader)); assert( sizeof(zMagicHeader)==16 ); - put2byte(&data[16], pBt->pageSize); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); data[18] = 1; data[19] = 1; assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); data[20] = (u8)(pBt->pageSize - pBt->usableSize); data[21] = 64; @@ -46037,17 +47812,31 @@ } p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); if( p->inTrans>pBt->inTransaction ){ pBt->inTransaction = p->inTrans; } -#ifndef SQLITE_OMIT_SHARED_CACHE if( wrflag ){ + MemPage *pPage1 = pBt->pPage1; +#ifndef SQLITE_OMIT_SHARED_CACHE assert( !pBt->pWriter ); pBt->pWriter = p; pBt->isExclusive = (u8)(wrflag>1); +#endif + + /* If the db-size header field is incorrect (as it may be if an old + ** client has been writing the database file), update it now. Doing + ** this sooner rather than later means the database size can safely + ** re-read the database size from page 1 if a savepoint or transaction + ** rollback occurs within the transaction. + */ + if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pPage1->aData[28], pBt->nPage); + } + } } -#endif } trans_begun: if( rc==SQLITE_OK && wrflag ){ @@ -46569,20 +48358,21 @@ ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p){ - BtShared *pBt = p->pBt; + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; sqlite3BtreeEnter(p); btreeIntegrity(p); /* If the handle has a write-transaction open, commit the shared-btrees ** transaction and set the shared state to TRANS_READ. */ if( p->inTrans==TRANS_WRITE ){ int rc; + BtShared *pBt = p->pBt; assert( pBt->inTransaction==TRANS_WRITE ); assert( pBt->nTransaction>0 ); rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); @@ -46782,13 +48572,15 @@ rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); if( rc==SQLITE_OK ){ if( iSavepoint<0 && pBt->initiallyEmpty ) pBt->nPage = 0; rc = newDatabase(pBt); pBt->nPage = get4byte(28 + pBt->pPage1->aData); - if( pBt->nPage==0 ){ - sqlite3PagerPagecount(pBt->pPager, (int*)&pBt->nPage); - } + + /* The database size was written into the offset 28 of the header + ** when the transaction started, so we know that the value at offset + ** 28 is nonzero. */ + assert( pBt->nPage>0 ); } sqlite3BtreeLeave(p); } return rc; } @@ -47880,13 +49672,13 @@ c = +1; } pCur->validNKey = 1; pCur->info.nKey = nCellKey; }else{ - /* The maximum supported page-size is 32768 bytes. This means that + /* The maximum supported page-size is 65536 bytes. This means that ** the maximum number of record bytes stored on an index B-Tree - ** page is at most 8198 bytes, which may be stored as a 2-byte + ** page is less than 16384 bytes and may be stored as a 2-byte ** varint. This information is used to attempt to avoid parsing ** the entire cell by checking for the cases where the record is ** stored entirely within the b-tree page by inspecting the first ** 2 bytes of the cell. */ @@ -48245,10 +50037,14 @@ } if( k==0 ){ if( !pPrevTrunk ){ memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); } }else{ /* The trunk page is required by the caller but it contains ** pointers to free-list leaves. The first leaf becomes a trunk @@ -48551,11 +50347,11 @@ BtShared *pBt = pPage->pBt; CellInfo info; Pgno ovflPgno; int rc; int nOvfl; - u16 ovflPageSize; + u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ @@ -48776,11 +50572,11 @@ ** ** "sz" must be the number of bytes in the cell. */ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ int i; /* Loop counter */ - int pc; /* Offset to cell content of cell being deleted */ + u32 pc; /* Offset to cell content of cell being deleted */ u8 *data; /* pPage->aData */ u8 *ptr; /* Used to move bytes around within data[] */ int rc; /* The return code */ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ @@ -48794,11 +50590,11 @@ ptr = &data[pPage->cellOffset + 2*idx]; pc = get2byte(ptr); hdr = pPage->hdrOffset; testcase( pc==get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ *pRC = SQLITE_CORRUPT_BKPT; return; } rc = freeSpace(pPage, pc, sz); if( rc ){ @@ -48851,11 +50647,11 @@ int nSkip = (iChild ? 4 : 0); if( *pRC ) return; assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nOverflow<=ArraySize(pPage->aOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* The cell should normally be sized correctly. However, when moving a ** malformed cell from a leaf page to an interior page, if the cell size ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size @@ -48931,16 +50727,16 @@ const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=5460 ); + assert( nCell>=0 && nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); /* Check that the page has just been zeroed by zeroPage() */ assert( pPage->nCell==0 ); - assert( get2byte(&data[hdr+5])==nUsable ); + assert( get2byteNotZero(&data[hdr+5])==nUsable ); pCellptr = &data[pPage->cellOffset + nCell*2]; cellbody = nUsable; for(i=nCell-1; i>=0; i--){ pCellptr -= 2; @@ -49002,10 +50798,11 @@ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); + /* This error condition is now caught prior to reaching this function */ if( pPage->nCell<=0 ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell ** may be inserted. If both these operations are successful, proceed. @@ -49331,11 +51128,11 @@ ** In this case, temporarily copy the cell into the aOvflSpace[] ** buffer. It will be copied out again as soon as the aSpace[] buffer ** is allocated. */ if( pBt->secureDelete ){ int iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); - if( (iOff+szNew[i])>pBt->usableSize ){ + if( (iOff+szNew[i])>(int)pBt->usableSize ){ rc = SQLITE_CORRUPT_BKPT; memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; }else{ memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); @@ -49410,11 +51207,11 @@ u8 *pTemp; assert( nCellpageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iSpace1<=pBt->pageSize ); memcpy(pTemp, apDiv[i], sz); apCell[nCell] = pTemp+leafCorrection; assert( leafCorrection==0 || leafCorrection==4 ); szCell[nCell] = szCell[nCell] - leafCorrection; @@ -49656,11 +51453,11 @@ assert(leafCorrection==4); sz = cellSizePtr(pParent, pCell); } } iOvflSpace += sz; - assert( sz<=pBt->pageSize/4 ); + assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace<=pBt->pageSize ); insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -50293,15 +52090,16 @@ ** flags might not work: ** ** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys ** BTREE_ZERODATA Used for SQL indices */ -static int btreeCreateTable(Btree *p, int *piTable, int flags){ +static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ BtShared *pBt = p->pBt; MemPage *pRoot; Pgno pgnoRoot; int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( !pBt->readOnly ); @@ -50416,12 +52214,18 @@ rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); if( rc ) return rc; } #endif assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - zeroPage(pRoot, flags | PTF_LEAF); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); *piTable = (int)pgnoRoot; return SQLITE_OK; } SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ int rc; @@ -50900,11 +52704,11 @@ #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); } #endif - if( n>pCheck->pBt->usableSize/4-2 ){ + if( n>(int)pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, zContext, "freelist leaf count too big on page %d", iPage); N--; }else{ for(i=0; ihdrOffset; hit = sqlite3PageMalloc( pBt->pageSize ); if( hit==0 ){ pCheck->mallocFailed = 1; }else{ - u16 contentOffset = get2byte(&data[hdr+5]); + int contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); nCell = get2byte(&data[hdr+3]); cellStart = hdr + 12 - 4*pPage->leaf; for(i=0; i=usableSize ){ + if( (int)(pc+size-1)>=usableSize ){ checkAppendMsg(pCheck, 0, "Corruption detected in cell %d on page %d",i,iPage); }else{ for(j=pc+size-1; j>=pc; j--) hit[j]++; } @@ -51217,10 +53021,11 @@ i = PENDING_BYTE_PAGE(pBt); if( i<=sCheck.nPage ){ sCheck.anRef[i] = 1; } sqlite3StrAccumInit(&sCheck.errMsg, zErr, sizeof(zErr), 20000); + sCheck.errMsg.useMalloc = 2; /* Check the integrity of the freelist */ checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); @@ -51315,10 +53120,33 @@ SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); return (p && (p->inTrans==TRANS_WRITE)); } +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +*/ +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + /* ** Return non-zero if a read (or write) transaction is active. */ SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){ assert( p ); @@ -51354,11 +53182,11 @@ */ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); if( !pBt->pSchema && nBytes ){ - pBt->pSchema = sqlite3MallocZero(nBytes); + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); pBt->xFreeSchema = xFree; } sqlite3BtreeLeave(p); return pBt->pSchema; } @@ -51459,12 +53287,11 @@ ** sqlite3BtreePutData()). */ SQLITE_PRIVATE void sqlite3BtreeCacheOverflow(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - assert(!pCur->isIncrblobHandle); - assert(!pCur->aOverflow); + invalidateOverflowCache(pCur); pCur->isIncrblobHandle = 1; } #endif /* @@ -51653,11 +53480,14 @@ sqlite3Error( pDestDb, SQLITE_ERROR, "source and destination must be distinct" ); p = 0; }else { - /* Allocate space for a new sqlite3_backup object */ + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3_malloc(sizeof(sqlite3_backup)); if( !p ){ sqlite3Error(pDestDb, SQLITE_NOMEM, 0); } } @@ -51723,10 +53553,19 @@ ** page sizes of the source and destination differ. */ if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ rc = SQLITE_READONLY; } + +#ifdef SQLITE_HAS_CODEC + /* Backup is not possible if the page size of the destination is changing + ** a a codec is in use. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ + rc = SQLITE_READONLY; + } +#endif /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. */ @@ -52027,10 +53866,13 @@ if( p->pDestDb ){ sqlite3_mutex_leave(p->pDestDb->mutex); } sqlite3BtreeLeave(p->pSrc); if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ sqlite3_free(p); } sqlite3_mutex_leave(mutex); return rc; } @@ -52278,10 +54120,13 @@ return SQLITE_NOMEM; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; pMem->flags |= MEM_Term; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif } return SQLITE_OK; } @@ -52398,11 +54243,11 @@ memset(&ctx, 0, sizeof(ctx)); ctx.s.flags = MEM_Null; ctx.s.db = pMem->db; ctx.pMem = pMem; ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); sqlite3DbFree(pMem->db, pMem->zMalloc); memcpy(pMem, &ctx.s, sizeof(ctx.s)); rc = ctx.isError; } @@ -52511,17 +54356,13 @@ return pMem->u.i; }else if( flags & MEM_Real ){ return doubleToInt64(pMem->r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - return 0; - } - assert( pMem->z ); - sqlite3Atoi64(pMem->z, &value); + assert( pMem->z || pMem->n==0 ); + testcase( pMem->z==0 ); + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ return 0; } } @@ -52540,18 +54381,11 @@ }else if( pMem->flags & MEM_Int ){ return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ double val = (double)0; - pMem->flags |= MEM_Str; - if( sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8) - || sqlite3VdbeMemNulTerminate(pMem) ){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - return (double)0; - } - assert( pMem->z ); - sqlite3AtoF(pMem->z, &val); + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); return val; }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; } @@ -52620,34 +54454,34 @@ ** Every effort is made to force the conversion, even if the input ** is a string that does not look completely like a number. Convert ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - int rc; - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ); - assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3VdbeChangeEncoding(pMem, SQLITE_UTF8); - if( rc ) return rc; - rc = sqlite3VdbeMemNulTerminate(pMem); - if( rc ) return rc; - if( sqlite3Atoi64(pMem->z, &pMem->u.i) ){ - MemSetTypeFlag(pMem, MEM_Int); - }else{ - pMem->r = sqlite3VdbeRealValue(pMem); - MemSetTypeFlag(pMem, MEM_Real); - sqlite3VdbeIntegerAffinity(pMem); - } + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + MemSetTypeFlag(pMem, MEM_Int); + }else{ + pMem->r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + sqlite3VdbeIntegerAffinity(pMem); + } + } + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob); return SQLITE_OK; } /* ** Delete any previous value and set the value stored in *pMem to NULL. */ SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ if( pMem->flags & MEM_Frame ){ - sqlite3VdbeFrameDelete(pMem->u.pFrame); + VdbeFrame *pFrame = pMem->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } if( pMem->flags & MEM_RowSet ){ sqlite3RowSetClear(pMem->u.pRowSet); } MemSetTypeFlag(pMem, MEM_Null); @@ -52739,10 +54573,32 @@ return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; } return 0; } +#ifdef SQLITE_DEBUG +/* +** This routine prepares a memory cell for modication by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to make sure shallow +** copies are not misused. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; + for(i=1, pX=&pVdbe->aMem[1]; i<=pVdbe->nMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + pX->flags |= MEM_Invalid; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ + /* ** Size of struct Mem not including the Mem.zMalloc member. */ #define MEMCELLSIZE (size_t)(&(((Mem *)0)->zMalloc)) @@ -53107,11 +54963,11 @@ assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ return 0; } } - sqlite3VdbeMemNulTerminate(pVal); + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-59893-45467 */ }else{ assert( (pVal->flags&MEM_Blob)==0 ); sqlite3VdbeMemStringify(pVal, enc); assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); } @@ -53155,51 +55011,68 @@ sqlite3_value **ppVal /* Write the new value here */ ){ int op; char *zVal = 0; sqlite3_value *pVal = 0; + int negInt = 1; + const char *zNeg = ""; if( !pExpr ){ *ppVal = 0; return SQLITE_OK; } op = pExpr->op; - /* op can only be TK_REGISTER is we have compiled with SQLITE_ENABLE_STAT2. + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2. ** The ifdef here is to enable us to achieve 100% branch test coverage even ** when SQLITE_ENABLE_STAT2 is omitted. */ #ifdef SQLITE_ENABLE_STAT2 if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; #endif + + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. + */ + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ pVal = sqlite3ValueNew(db); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ - sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue); + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3DbStrDup(db, pExpr->u.zToken); + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); if( op==TK_FLOAT ) pVal->type = SQLITE_FLOAT; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } + if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; if( enc!=SQLITE_UTF8 ){ sqlite3VdbeChangeEncoding(pVal, enc); } }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ + sqlite3VdbeMemNumerify(pVal); pVal->u.i = -1 * pVal->u.i; /* (double)-1 In case of SQLITE_OMIT_FLOATING_POINT... */ pVal->r = (double)-1 * pVal->r; + sqlite3ValueApplyAffinity(pVal, affinity, enc); } } #ifndef SQLITE_OMIT_BLOB_LITERAL else if( op==TK_BLOB ){ int nVal; @@ -53840,47 +55713,55 @@ if( ALWAYS(pDef) && (pDef->flags & SQLITE_FUNC_EPHEM)!=0 ){ sqlite3DbFree(db, pDef); } } +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* ** Delete a P4 value if necessary. */ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ + assert( db ); switch( p4type ){ case P4_REAL: case P4_INT64: - case P4_MPRINTF: case P4_DYNAMIC: case P4_KEYINFO: case P4_INTARRAY: case P4_KEYINFO_HANDOFF: { sqlite3DbFree(db, p4); break; + } + case P4_MPRINTF: { + if( db->pnBytesFreed==0 ) sqlite3_free(p4); + break; } case P4_VDBEFUNC: { VdbeFunc *pVdbeFunc = (VdbeFunc *)p4; freeEphemeralFunction(db, pVdbeFunc->pFunc); - sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); + if( db->pnBytesFreed==0 ) sqlite3VdbeDeleteAuxData(pVdbeFunc, 0); sqlite3DbFree(db, pVdbeFunc); break; } case P4_FUNCDEF: { freeEphemeralFunction(db, (FuncDef*)p4); break; } case P4_MEM: { - sqlite3ValueFree((sqlite3_value*)p4); + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + Mem *p = (Mem*)p4; + sqlite3DbFree(db, p->zMalloc); + sqlite3DbFree(db, p); + } break; } case P4_VTAB : { - sqlite3VtabUnlock((VTable *)p4); - break; - } - case P4_SUBPROGRAM : { - sqlite3VdbeProgramDelete(db, (SubProgram *)p4, 1); + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } } } } @@ -53902,38 +55783,18 @@ } sqlite3DbFree(db, aOp); } /* -** Decrement the ref-count on the SubProgram structure passed as the -** second argument. If the ref-count reaches zero, free the structure. -** -** The array of VDBE opcodes stored as SubProgram.aOp is freed if -** either the ref-count reaches zero or parameter freeop is non-zero. -** -** Since the array of opcodes pointed to by SubProgram.aOp may directly -** or indirectly contain a reference to the SubProgram structure itself. -** By passing a non-zero freeop parameter, the caller may ensure that all -** SubProgram structures and their aOp arrays are freed, even when there -** are such circular references. +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. */ -SQLITE_PRIVATE void sqlite3VdbeProgramDelete(sqlite3 *db, SubProgram *p, int freeop){ - if( p ){ - assert( p->nRef>0 ); - if( freeop || p->nRef==1 ){ - Op *aOp = p->aOp; - p->aOp = 0; - vdbeFreeOpArray(db, aOp, p->nOp); - p->nOp = 0; - } - p->nRef--; - if( p->nRef==0 ){ - sqlite3DbFree(db, p); - } - } -} - +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; +} /* ** Change N opcodes starting at addr to No-ops. */ SQLITE_PRIVATE void sqlite3VdbeChangeToNoop(Vdbe *p, int addr, int N){ @@ -54006,11 +55867,11 @@ KeyInfo *pKeyInfo; int nField, nByte; nField = ((KeyInfo*)zP4)->nField; nByte = sizeof(*pKeyInfo) + (nField-1)*sizeof(pKeyInfo->aColl[0]) + nField; - pKeyInfo = sqlite3Malloc( nByte ); + pKeyInfo = sqlite3DbMallocRaw(0, nByte); pOp->p4.pKeyInfo = pKeyInfo; if( pKeyInfo ){ u8 *aSortOrder; memcpy((char*)pKeyInfo, zP4, nByte - nField); aSortOrder = pKeyInfo->aSortOrder; @@ -54270,10 +56131,16 @@ static void releaseMemArray(Mem *p, int N){ if( p && N ){ Mem *pEnd; sqlite3 *db = p->db; u8 malloc_failed = db->mallocFailed; + if( db->pnBytesFreed ){ + for(pEnd=&p[N]; pzMalloc); + } + return; + } for(pEnd=&p[N]; pflags = MEM_Int; pMem->u.i = pOp->p2; /* P2 */ pMem->type = SQLITE_INTEGER; pMem++; - if( p->explain==1 ){ - pMem->flags = MEM_Int; - pMem->u.i = pOp->p3; /* P3 */ - pMem->type = SQLITE_INTEGER; - pMem++; - } + pMem->flags = MEM_Int; + pMem->u.i = pOp->p3; /* P3 */ + pMem->type = SQLITE_INTEGER; + pMem++; if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -54505,11 +56370,11 @@ pMem->flags = MEM_Null; /* Comment */ pMem->type = SQLITE_NULL; } } - p->nResColumn = 8 - 5*(p->explain-1); + p->nResColumn = 8 - 4*(p->explain-1); p->rc = SQLITE_OK; rc = SQLITE_ROW; } return rc; } @@ -54812,10 +56677,15 @@ } } if( p->aMem ){ releaseMemArray(&p->aMem[1], p->nMem); } + while( p->pDelFrame ){ + VdbeFrame *pDel = p->pDelFrame; + p->pDelFrame = pDel->pParent; + sqlite3VdbeFrameDelete(pDel); + } } /* ** Clean up the VM after execution. ** @@ -54919,26 +56789,27 @@ ** be done before determining whether a master journal file is ** required, as an xSync() callback may add an attached database ** to the transaction. */ rc = sqlite3VtabSync(db, &p->zErrMsg); - if( rc!=SQLITE_OK ){ - return rc; - } /* This loop determines (a) if the commit hook should be invoked and ** (b) how many database files have open write transactions, not ** including the temp database. (b) is important because if more than ** one database file has an open write transaction, a master journal ** file is required for an atomic commit. */ - for(i=0; inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; if( i!=1 ) nTrans++; + rc = sqlite3PagerExclusiveLock(sqlite3BtreePager(pBt)); } + } + if( rc!=SQLITE_OK ){ + return rc; } /* If there are any write-transactions at all, invoke the commit hook */ if( needXcommit && db->xCommitCallback ){ rc = db->xCommitCallback(db->pCommitArg); @@ -55027,13 +56898,14 @@ */ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ char const *zFile = sqlite3BtreeGetJournalname(pBt); - if( zFile==0 || zFile[0]==0 ){ + if( zFile==0 ){ continue; /* Ignore TEMP and :memory: databases */ } + assert( zFile[0]!=0 ); if( !needSync && !sqlite3BtreeSyncDisabled(pBt) ){ needSync = 1; } rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); offset += sqlite3Strlen30(zFile)+1; @@ -55074,10 +56946,11 @@ if( pBt ){ rc = sqlite3BtreeCommitPhaseOne(pBt, zMaster); } } sqlite3OsCloseFree(pMaster); + assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ sqlite3DbFree(db, zMaster); return rc; } @@ -55332,12 +57205,21 @@ mrc = p->rc & 0xff; assert( p->rc!=SQLITE_IOERR_BLOCKED ); /* This error no longer exists */ isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; if( isSpecialError ){ - /* If the query was read-only, we need do no rollback at all. Otherwise, - ** proceed with the special handling. + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occured while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. */ if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ eStatementOp = SAVEPOINT_ROLLBACK; }else{ @@ -55601,10 +57483,34 @@ } pAux->pAux = 0; } } } + +/* +** Free all memory associated with the Vdbe passed as the second argument. +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection. +*/ +SQLITE_PRIVATE void sqlite3VdbeDeleteObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aVar, p->nVar); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aLabel); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); + sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p); +} /* ** Delete an entire VDBE. */ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ @@ -55619,20 +57525,13 @@ db->pVdbe = p->pNext; } if( p->pNext ){ p->pNext->pPrev = p->pPrev; } - releaseMemArray(p->aVar, p->nVar); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aLabel); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); p->magic = VDBE_MAGIC_DEAD; - sqlite3DbFree(db, p->pFree); p->db = 0; - sqlite3DbFree(db, p); + sqlite3VdbeDeleteObject(db, p); } /* ** Make sure the cursor p is ready to read or write the row to which it ** was last positioned. Return an error code if an OOM fault or I/O error @@ -55654,15 +57553,12 @@ #endif assert( p->isTable ); rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; p->lastRowid = p->movetoTarget; - p->rowidIsValid = ALWAYS(res==0) ?1:0; - if( NEVER(res<0) ){ - rc = sqlite3BtreeNext(p->pCursor, &res); - if( rc ) return rc; - } + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif p->deferredMoveto = 0; p->cacheStatus = CACHE_STALE; @@ -56469,10 +58365,12 @@ ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ rc = SQLITE_OK; }else{ Vdbe *v = (Vdbe*)pStmt; sqlite3 *db = v->db; #if SQLITE_THREADSAFE @@ -56545,11 +58443,11 @@ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ sqlite3VdbeMemExpandBlob(p); p->flags &= ~MEM_Str; p->flags |= MEM_Blob; - return p->z; + return p->n ? p->z : 0; }else{ return sqlite3_value_text(pVal); } } SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ @@ -56802,11 +58700,11 @@ /* Invoke the profile callback if there is one */ if( rc!=SQLITE_ROW && db->xProfile && !db->init.busy && p->zSql ){ sqlite3_int64 iNow; sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, iNow - p->startTime); + db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); } #endif if( rc==SQLITE_DONE ){ assert( p->rc==SQLITE_OK ); @@ -56899,10 +58797,16 @@ } /* ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. +** +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. */ SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ assert( p && p->pFunc ); return p->s.db; } @@ -57108,12 +59012,11 @@ ** sqlite3_column_text() ** sqlite3_column_text16() ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** -** But not for sqlite3_column_blob(), which never calls malloc(). +** sqiite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { /* If malloc() failed during an encoding conversion within an ** sqlite3_column_XXX API, then set the return code of the statement to @@ -57377,10 +59280,16 @@ pVar->flags = MEM_Null; sqlite3Error(p->db, SQLITE_OK, 0); /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. */ if( p->isPrepareV2 && ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) ){ p->expired = 1; @@ -57413,10 +59322,12 @@ } sqlite3Error(p->db, rc, 0); rc = sqlite3ApiExit(p->db, rc); } sqlite3_mutex_leave(p->db->mutex); + }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ + xDel((void*)zData); } return rc; } @@ -57654,10 +59565,18 @@ ** the statement in the first place. */ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->db : 0; } + +/* +** Return true if the prepared statement is guaranteed to not modify the +** database. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; +} /* ** Return a pointer to the next prepared statement after pStmt associated ** with database connection pDb. If pStmt is NULL, return the first ** prepared statement for the database connection. Return NULL if there @@ -57874,10 +59793,21 @@ ** of the code in this file is, therefore, important. See other comments ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. */ +/* +** Invoke this macro on memory cells just prior to changing the +** value of the cell. This macro verifies that shallow copies are +** not misused. +*/ +#ifdef SQLITE_DEBUG +# define memAboutToChange(P,M) sqlite3VdbeMemPrepareToChange(P,M) +#else +# define memAboutToChange(P,M) +#endif + /* ** The following global variable is incremented every time a cursor ** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test ** procedures use this information to make sure that indices are ** working correctly. This variable has no function other than to @@ -58066,35 +59996,21 @@ ** looks like a number, convert it into a number. If it does not ** look like a number, leave it alone. */ static void applyNumericAffinity(Mem *pRec){ if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - int realnum; + double rValue; + i64 iValue; u8 enc = pRec->enc; - sqlite3VdbeMemNulTerminate(pRec); - if( (pRec->flags&MEM_Str) && sqlite3IsNumber(pRec->z, &realnum, enc) ){ - i64 value; - char *zUtf8 = pRec->z; -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - assert( pRec->db ); - zUtf8 = sqlite3Utf16to8(pRec->db, pRec->z, pRec->n, enc); - if( !zUtf8 ) return; - } -#endif - if( !realnum && sqlite3Atoi64(zUtf8, &value) ){ - pRec->u.i = value; - MemSetTypeFlag(pRec, MEM_Int); - }else{ - sqlite3AtoF(zUtf8, &pRec->r); - MemSetTypeFlag(pRec, MEM_Real); - } -#ifndef SQLITE_OMIT_UTF16 - if( enc!=SQLITE_UTF8 ){ - sqlite3DbFree(pRec->db, zUtf8); - } -#endif + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; } } } /* @@ -58142,17 +60058,17 @@ /* ** Try to convert the type of a function argument or a result column ** into a numeric representation. Use either INTEGER or REAL whichever ** is appropriate. But only do the conversion if it is possible without ** loss of information and return the revised type of the argument. -** -** This is an EXPERIMENTAL api and is subject to change or removal. */ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); - sqlite3VdbeMemStoreType(pMem); + if( pMem->type==SQLITE_TEXT ){ + applyNumericAffinity(pMem); + sqlite3VdbeMemStoreType(pMem); + } return pMem->type; } /* ** Exported version of applyAffinity(). This one works on sqlite3_value*, @@ -58417,10 +60333,24 @@ for(p=db->pSavepoint; p; p=p->pNext) n++; assert( n==(db->nSavepoint + db->isTransactionSavepoint) ); return 1; } #endif + +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +static void importVtabErrMsg(Vdbe *p, sqlite3_vtab *pVtab){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; +} + /* ** Execute as much of a VDBE program as we can then return. ** ** sqlite3VdbeMakeReady() must be called before this routine in order to @@ -58969,38 +60899,44 @@ assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); sqlite3VdbeMemReleaseExternal(pOut); pOut->flags = MEM_Int; } /* Sanity checking on other operands */ #ifdef SQLITE_DEBUG if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); assert( pOp->p1<=p->nMem ); + assert( memIsValid(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + assert( memIsValid(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + assert( memIsValid(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=p->nMem ); + memAboutToChange(p, &aMem[pOp->p3]); } #endif switch( pOp->opcode ){ @@ -59058,10 +60994,11 @@ ** and then jump to address P2. */ case OP_Gosub: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); + memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; pIn1->u.i = pc; REGISTER_TRACE(pOp->p1, pIn1); pc = pOp->p2 - 1; break; @@ -59265,15 +61202,11 @@ /* Opcode: Blob P1 P2 * P4 ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. This instruction is not coded directly -** by the compiler. Instead, the compiler layer specifies -** an OP_HexBlob opcode, with the hex string representation of -** the blob as P4. This opcode is transformed to an OP_Blob -** the first time it is executed. +** blob in register P2. */ case OP_Blob: { /* out2-prerelease */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); pOut->enc = encoding; @@ -59327,10 +61260,12 @@ pIn1 = &aMem[u.ac.p1]; pOut = &aMem[u.ac.p2]; while( u.ac.n-- ){ assert( pOut<=&aMem[p->nMem] ); assert( pIn1<=&aMem[p->nMem] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); u.ac.zMalloc = pOut->zMalloc; pOut->zMalloc = 0; sqlite3VdbeMemMove(pOut, pIn1); pIn1->zMalloc = u.ac.zMalloc; REGISTER_TRACE(u.ac.p2++, pOut); @@ -59372,10 +61307,13 @@ case OP_SCopy: { /* in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; +#endif REGISTER_TRACE(pOp->p2, pOut); break; } /* Opcode: ResultRow P1 P2 * * * @@ -59432,10 +61370,14 @@ ** and have an assigned type. The results are de-ephemeralized as ** as side effect. */ u.ad.pMem = p->pResultSet = &aMem[pOp->p1]; for(u.ad.i=0; u.ad.ip2; u.ad.i++){ + assert( memIsValid(&u.ad.pMem[u.ad.i]) ); + Deephemeralize(&u.ad.pMem[u.ad.i]); + assert( (u.ad.pMem[u.ad.i].flags & MEM_Ephem)==0 + || (u.ad.pMem[u.ad.i].flags & (MEM_Str|MEM_Blob))==0 ); sqlite3VdbeMemNulTerminate(&u.ad.pMem[u.ad.i]); sqlite3VdbeMemStoreType(&u.ad.pMem[u.ad.i]); REGISTER_TRACE(pOp->p1+u.ad.i, &u.ad.pMem[u.ad.i]); } if( db->mallocFailed ) goto no_mem; @@ -59663,16 +61605,21 @@ #endif /* local variables moved into u.ag */ u.ag.n = pOp->p5; u.ag.apVal = p->apArg; assert( u.ag.apVal || u.ag.n==0 ); + assert( pOp->p3>0 && pOp->p3<=p->nMem ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); assert( u.ag.n==0 || (pOp->p2>0 && pOp->p2+u.ag.n<=p->nMem+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+u.ag.n ); u.ag.pArg = &aMem[pOp->p2]; for(u.ag.i=0; u.ag.ip2+u.ag.i, u.ag.pArg); } assert( pOp->p4type==P4_FUNCDEF || pOp->p4type==P4_VDBEFUNC ); @@ -59682,12 +61629,10 @@ }else{ u.ag.ctx.pVdbeFunc = (VdbeFunc*)pOp->p4.pVdbeFunc; u.ag.ctx.pFunc = u.ag.ctx.pVdbeFunc->pFunc; } - assert( pOp->p3>0 && pOp->p3<=p->nMem ); - pOut = &aMem[pOp->p3]; u.ag.ctx.s.flags = MEM_Null; u.ag.ctx.s.db = db; u.ag.ctx.s.xDel = 0; u.ag.ctx.s.zMalloc = 0; @@ -59703,11 +61648,11 @@ assert( pOp>aOp ); assert( pOp[-1].p4type==P4_COLLSEQ ); assert( pOp[-1].opcode==OP_CollSeq ); u.ag.ctx.pColl = pOp[-1].p4.pColl; } - (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); + (*u.ag.ctx.pFunc->xFunc)(&u.ag.ctx, u.ag.n, u.ag.apVal); /* IMP: R-24505-23230 */ if( db->mallocFailed ){ /* Even though a malloc() has failed, the implementation of the ** user function may have called an sqlite3_result_XXX() function ** to return a value. The following call releases any resources ** associated with such a value. @@ -59755,11 +61700,11 @@ ** If either input is NULL, the result is NULL. */ /* Opcode: ShiftLeft P1 P2 P3 * * ** ** Shift the integer value in register P2 to the left by the -** number of bits specified by the integer in regiser P1. +** number of bits specified by the integer in register P1. ** Store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: ShiftRight P1 P2 P3 * * ** @@ -59805,10 +61750,11 @@ ** ** To force any register to be an integer, just add 0. */ case OP_AddImm: { /* in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); sqlite3VdbeMemIntegerify(pIn1); pIn1->u.i += pOp->p2; break; } @@ -59864,10 +61810,11 @@ ** ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToText: { /* same as TK_TO_TEXT, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( pIn1->flags & MEM_Null ) break; assert( MEM_Str==(MEM_Blob>>3) ); pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); rc = ExpandBlob(pIn1); @@ -59910,20 +61857,18 @@ ** ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & (MEM_Null|MEM_Int|MEM_Real))==0 ){ - sqlite3VdbeMemNumerify(pIn1); - } + sqlite3VdbeMemNumerify(pIn1); break; } #endif /* SQLITE_OMIT_CAST */ /* Opcode: ToInt P1 * * * * ** -** Force the value in register P1 be an integer. If +** Force the value in register P1 to be an integer. If ** The value is currently a real number, drop its fractional part. ** If the value is text or blob, try to convert it to an integer using the ** equivalent of atoi() and store 0 if no such conversion is possible. ** ** A NULL value is not changed by this routine. It remains NULL. @@ -59946,10 +61891,11 @@ ** ** A NULL value is not changed by this routine. It remains NULL. */ case OP_ToReal: { /* same as TK_TO_REAL, in1 */ pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); if( (pIn1->flags & MEM_Null)==0 ){ sqlite3VdbeMemRealify(pIn1); } break; } @@ -59960,11 +61906,11 @@ ** Compare the values in register P1 and P3. If reg(P3)=0; break; } if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); MemSetTypeFlag(pOut, MEM_Int); pOut->u.i = u.ai.res; REGISTER_TRACE(pOp->p2, pOut); }else if( u.ai.res ){ pc = pOp->p2-1; @@ -60121,12 +62068,12 @@ break; } /* Opcode: Compare P1 P2 P3 P4 * ** -** Compare to vectors of registers in reg(P1)..reg(P1+P3-1) (all this -** one "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of ** the comparison for use by the next OP_Jump instruct. ** ** P4 is a KeyInfo structure that defines collating sequences and sort ** orders for the comparison. The permutation applies to registers ** only. The KeyInfo elements are used sequentially. @@ -60164,10 +62111,12 @@ assert( u.aj.p2>0 && u.aj.p2+u.aj.n<=p->nMem+1 ); } #endif /* SQLITE_DEBUG */ for(u.aj.i=0; u.aj.inField ); u.aj.pColl = u.aj.pKeyInfo->aColl[u.aj.i]; u.aj.bRev = u.aj.pKeyInfo->aSortOrder[u.aj.i]; @@ -60395,10 +62344,11 @@ u.am.pC = 0; memset(&u.am.sMem, 0, sizeof(u.am.sMem)); assert( u.am.p1nCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.am.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.am.pDest); MemSetTypeFlag(u.am.pDest, MEM_Null); u.am.zRec = 0; /* This block sets the variable u.am.payloadSize to be the total number of ** bytes in the record. @@ -60442,10 +62392,11 @@ assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } }else if( u.am.pC->pseudoTableReg>0 ){ u.am.pReg = &aMem[u.am.pC->pseudoTableReg]; assert( u.am.pReg->flags & MEM_Blob ); + assert( memIsValid(u.am.pReg) ); u.am.payloadSize = u.am.pReg->n; u.am.zRec = u.am.pReg->z; u.am.pC->cacheStatus = (pOp->p5&OPFLAG_CLEARCACHE) ? CACHE_STALE : p->cacheCtr; assert( u.am.payloadSize==0 || u.am.zRec!=0 ); }else{ @@ -60666,25 +62617,23 @@ assert( u.an.zAffinity!=0 ); assert( u.an.zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (u.an.cAff = *(u.an.zAffinity++))!=0 ){ assert( pIn1 <= &p->aMem[p->nMem] ); + assert( memIsValid(pIn1) ); ExpandBlob(pIn1); applyAffinity(pIn1, u.an.cAff, encoding); pIn1++; } break; } /* Opcode: MakeRecord P1 P2 P3 P4 * ** -** Convert P2 registers beginning with P1 into a single entry -** suitable for use as a data record in a database table or as a key -** in an index. The details of the format are irrelevant as long as -** the OP_Column opcode can decode the record later. -** Refer to source code comments for the details of the record -** format. +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. ** ** P4 may be a string that is P2 characters long. The nth character of the ** string indicates the column affinity that should be used for the nth ** field of the index key. ** @@ -60736,15 +62685,21 @@ assert( u.ao.nField>0 && pOp->p2>0 && pOp->p2+u.ao.nField<=p->nMem+1 ); u.ao.pData0 = &aMem[u.ao.nField]; u.ao.nField = pOp->p2; u.ao.pLast = &u.ao.pData0[u.ao.nField-1]; u.ao.file_format = p->minWriteFileFormat; + + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); /* Loop through the elements that will make up the record to figure ** out how much space is required for the new record. */ for(u.ao.pRec=u.ao.pData0; u.ao.pRec<=u.ao.pLast; u.ao.pRec++){ + assert( memIsValid(u.ao.pRec) ); if( u.ao.zAffinity ){ applyAffinity(u.ao.pRec, u.ao.zAffinity[u.ao.pRec-u.ao.pData0], encoding); } if( u.ao.pRec->flags&MEM_Zero && u.ao.pRec->n>0 ){ sqlite3VdbeMemExpandBlob(u.ao.pRec); @@ -60775,12 +62730,10 @@ /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to ** sqlite3VdbeMemGrow() could clobber the value before it is used). */ - assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); - pOut = &aMem[pOp->p3]; if( sqlite3VdbeMemGrow(pOut, (int)u.ao.nByte, 0) ){ goto no_mem; } u.ao.zNewRecord = (u8 *)pOut->z; @@ -60949,10 +62902,11 @@ } } if( u.aq.p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetInternalSchema(db, 0); + db->flags = (db->flags | SQLITE_InternChanges); } } /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all ** savepoints nested inside of the savepoint being operated on. */ @@ -61339,10 +63293,12 @@ } if( pOp->p5 ){ assert( u.aw.p2>0 ); assert( u.aw.p2<=p->nMem ); pIn2 = &aMem[u.aw.p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); sqlite3VdbeMemIntegerify(pIn2); u.aw.p2 = (int)pIn2->u.i; /* The u.aw.p2 value always comes from a prior OP_CreateTable opcode and ** that opcode will always set the u.aw.p2 value to 2 or more or else fail. ** If there were a failure, the prepared statement would have halted @@ -61361,10 +63317,11 @@ } assert( pOp->p1>=0 ); u.aw.pCur = allocateCursor(p, pOp->p1, u.aw.nField, u.aw.iDb, 1); if( u.aw.pCur==0 ) goto no_mem; u.aw.pCur->nullRow = 1; + u.aw.pCur->isOrdered = 1; rc = sqlite3BtreeCursor(u.aw.pX, u.aw.p2, u.aw.wrFlag, u.aw.pKeyInfo, u.aw.pCur->pCursor); u.aw.pCur->pKeyInfo = u.aw.pKeyInfo; /* Since it performs no memory allocation or IO, the only values that ** sqlite3BtreeCursor() may return are SQLITE_EMPTY and SQLITE_OK. @@ -61413,11 +63370,11 @@ case OP_OpenAutoindex: case OP_OpenEphemeral: { #if 0 /* local variables moved into u.ax */ VdbeCursor *pCx; #endif /* local variables moved into u.ax */ - static const int openFlags = + static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TRANSIENT_DB; @@ -61424,25 +63381,25 @@ assert( pOp->p1>=0 ); u.ax.pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( u.ax.pCx==0 ) goto no_mem; u.ax.pCx->nullRow = 1; - rc = sqlite3BtreeFactory(db, 0, 1, SQLITE_DEFAULT_TEMP_CACHE_SIZE, openFlags, - &u.ax.pCx->pBt); + rc = sqlite3BtreeOpen(0, db, &u.ax.pCx->pBt, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ rc = sqlite3BtreeBeginTrans(u.ax.pCx->pBt, 1); } if( rc==SQLITE_OK ){ /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_ZERODATA flag before + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an INTKEY table). + ** automatically created table with root-page 1 (an BLOB_INTKEY table). */ if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_ZERODATA); + rc = sqlite3BtreeCreateTable(u.ax.pCx->pBt, &pgno, BTREE_BLOBKEY); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(u.ax.pCx->pBt, pgno, 1, (KeyInfo*)pOp->p4.z, u.ax.pCx->pCursor); u.ax.pCx->pKeyInfo = pOp->p4.pKeyInfo; @@ -61452,10 +63409,11 @@ }else{ rc = sqlite3BtreeCursor(u.ax.pCx->pBt, MASTER_ROOT, 1, 0, u.ax.pCx->pCursor); u.ax.pCx->isTable = 1; } } + u.ax.pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); u.ax.pCx->isIndex = !u.ax.pCx->isTable; break; } /* Opcode: OpenPseudo P1 P2 P3 * * @@ -61571,10 +63529,11 @@ assert( u.az.pC!=0 ); assert( u.az.pC->pseudoTableReg==0 ); assert( OP_SeekLe == OP_SeekLt+1 ); assert( OP_SeekGe == OP_SeekLt+2 ); assert( OP_SeekGt == OP_SeekLt+3 ); + assert( u.az.pC->isOrdered ); if( u.az.pC->pCursor!=0 ){ u.az.oc = pOp->opcode; u.az.pC->nullRow = 0; if( u.az.pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, @@ -61653,10 +63612,13 @@ assert( u.az.oc!=OP_SeekLe || u.az.r.flags==UNPACKED_INCRKEY ); assert( u.az.oc!=OP_SeekGe || u.az.r.flags==0 ); assert( u.az.oc!=OP_SeekLt || u.az.r.flags==0 ); u.az.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ipCursor, &u.az.r, 0, 0, &u.az.res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } @@ -61781,15 +63743,18 @@ assert( u.bb.pC->isTable==0 ); if( pOp->p4.i>0 ){ u.bb.r.pKeyInfo = u.bb.pC->pKeyInfo; u.bb.r.nField = (u16)pOp->p4.i; u.bb.r.aMem = pIn3; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iflags & MEM_Blob ); - ExpandBlob(pIn3); + assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ u.bb.pIdxKey = sqlite3VdbeRecordUnpack(u.bb.pC->pKeyInfo, pIn3->n, pIn3->z, u.bb.aTempRec, sizeof(u.bb.aTempRec)); if( u.bb.pIdxKey==0 ){ goto no_mem; } @@ -61880,10 +63845,13 @@ /* Populate the index search key. */ u.bc.r.pKeyInfo = u.bc.pCx->pKeyInfo; u.bc.r.nField = u.bc.nField + 1; u.bc.r.flags = UNPACKED_PREFIX_SEARCH; u.bc.r.aMem = u.bc.aMx; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iu.i; @@ -61902,11 +63870,11 @@ /* Opcode: NotExists P1 P2 P3 * * ** ** Use the content of register P3 as a integer key. If a record ** with that key does not exist in table of P1, then jump to P2. -** If the record does exist, then fall thru. The cursor is left +** If the record does exist, then fall through. The cursor is left ** pointing to the record if it exists. ** ** The difference between this operation and NotFound is that this ** operation assumes the key is an integer and that P1 is a table whereas ** NotFound assumes key is a blob constructed from MakeRecord and @@ -62060,11 +64028,13 @@ u.be.pMem = &u.be.pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ assert( pOp->p3<=p->nMem ); u.be.pMem = &aMem[pOp->p3]; + memAboutToChange(p, u.be.pMem); } + assert( memIsValid(u.be.pMem) ); REGISTER_TRACE(pOp->p3, u.be.pMem); sqlite3VdbeMemIntegerify(u.be.pMem); assert( (u.be.pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ if( u.be.pMem->u.i==MAX_ROWID || u.be.pC->useRandomRowid ){ @@ -62079,33 +64049,40 @@ #endif sqlite3BtreeSetCachedRowid(u.be.pC->pCursor, u.be.vuseRandomRowid ){ - /* IMPLEMENTATION-OF: R-48598-02938 If the largest ROWID is equal to the + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the ** largest possible integer (9223372036854775807) then the database - ** engine starts picking candidate ROWIDs at random until it finds one - ** that is not previously used. - */ + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ + /* on the first attempt, simply do one more than previous */ u.be.v = db->lastRowid; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ + u.be.v++; /* ensure non-zero */ u.be.cnt = 0; - do{ - if( u.be.cnt==0 && (u.be.v&0xffffff)==u.be.v ){ - u.be.v++; + while( ((rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, + 0, &u.be.res))==SQLITE_OK) + && (u.be.res==0) + && (++u.be.cnt<100)){ + /* collision - try another random rowid */ + sqlite3_randomness(sizeof(u.be.v), &u.be.v); + if( u.be.cnt<5 ){ + /* try "small" random rowids for the initial attempts */ + u.be.v &= 0xffffff; }else{ - sqlite3_randomness(sizeof(u.be.v), &u.be.v); - if( u.be.cnt<5 ) u.be.v &= 0xffffff; + u.be.v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ } - rc = sqlite3BtreeMovetoUnpacked(u.be.pC->pCursor, 0, (u64)u.be.v, 0, &u.be.res); - u.be.cnt++; - }while( u.be.cnt<100 && rc==SQLITE_OK && u.be.res==0 ); + u.be.v++; /* ensure non-zero */ + } if( rc==SQLITE_OK && u.be.res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; } + assert( u.be.v>0 ); /* EV: R-40812-03570 */ } u.be.pC->rowidIsValid = 0; u.be.pC->deferredMoveto = 0; u.be.pC->cacheStatus = CACHE_STALE; } @@ -62171,10 +64148,11 @@ int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ #endif /* local variables moved into u.bf */ u.bf.pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(u.bf.pData) ); u.bf.pC = p->apCsr[pOp->p1]; assert( u.bf.pC!=0 ); assert( u.bf.pC->pCursor!=0 ); assert( u.bf.pC->pseudoTableReg==0 ); assert( u.bf.pC->isTable ); @@ -62181,10 +64159,11 @@ REGISTER_TRACE(pOp->p2, u.bf.pData); if( pOp->opcode==OP_Insert ){ u.bf.pKey = &aMem[pOp->p3]; assert( u.bf.pKey->flags & MEM_Int ); + assert( memIsValid(u.bf.pKey) ); REGISTER_TRACE(pOp->p3, u.bf.pKey); u.bf.iKey = u.bf.pKey->u.i; }else{ assert( pOp->opcode==OP_InsertInt ); u.bf.iKey = pOp->p3; @@ -62332,10 +64311,11 @@ u32 n; i64 n64; #endif /* local variables moved into u.bh */ pOut = &aMem[pOp->p2]; + memAboutToChange(p, pOut); /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); u.bh.pC = p->apCsr[pOp->p1]; assert( u.bh.pC->isTable || pOp->opcode==OP_RowKey ); @@ -62416,13 +64396,11 @@ }else if( u.bi.pC->pVtabCursor ){ u.bi.pVtab = u.bi.pC->pVtabCursor->pVtab; u.bi.pModule = u.bi.pVtab->pModule; assert( u.bi.pModule->xRowid ); rc = u.bi.pModule->xRowid(u.bi.pC->pVtabCursor, &u.bi.v); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.bi.pVtab->zErrMsg; - u.bi.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.bi.pVtab); #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( u.bi.pC->pCursor!=0 ); rc = sqlite3VdbeCursorMoveto(u.bi.pC); if( rc ) goto abort_due_to_error; @@ -62676,10 +64654,13 @@ if( ALWAYS(u.bo.pCrsr!=0) ){ u.bo.r.pKeyInfo = u.bo.pC->pKeyInfo; u.bo.r.nField = (u16)pOp->p3; u.bo.r.flags = 0; u.bo.r.aMem = &aMem[pOp->p2]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; ideferredMoveto==0 ); @@ -62737,11 +64718,11 @@ ** If P5 is non-zero then the key value is increased by an epsilon ** prior to the comparison. This make the opcode work like IdxGT except ** that if the key from register P3 is a prefix of the key in the cursor, ** the result is false whereas it would be true with IdxGT. */ -/* Opcode: IdxLT P1 P2 P3 * P5 +/* Opcode: IdxLT P1 P2 P3 P4 P5 ** ** The P4 register values beginning with P3 form an unpacked index ** key that omits the ROWID. Compare this key value against the index ** that P1 is currently pointing to, ignoring the ROWID on the P1 index. ** @@ -62760,10 +64741,11 @@ #endif /* local variables moved into u.bq */ assert( pOp->p1>=0 && pOp->p1nCursor ); u.bq.pC = p->apCsr[pOp->p1]; assert( u.bq.pC!=0 ); + assert( u.bq.pC->isOrdered ); if( ALWAYS(u.bq.pC->pCursor!=0) ){ assert( u.bq.pC->deferredMoveto==0 ); assert( pOp->p5==0 || pOp->p5==1 ); assert( pOp->p4type==P4_INT32 ); u.bq.r.pKeyInfo = u.bq.pC->pKeyInfo; @@ -62772,10 +64754,13 @@ u.bq.r.flags = UNPACKED_INCRKEY | UNPACKED_IGNORE_ROWID; }else{ u.bq.r.flags = UNPACKED_IGNORE_ROWID; } u.bq.r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iopcode==OP_IdxLT ){ u.bq.res = -u.bq.res; }else{ assert( pOp->opcode==OP_IdxGE ); @@ -62875,10 +64860,12 @@ db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &u.bs.nChange : 0) ); if( pOp->p3 ){ p->nChange += u.bs.nChange; if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); aMem[pOp->p3].u.i += u.bs.nChange; } } break; } @@ -62918,13 +64905,13 @@ assert( (p->btreeMask & (1<p1))!=0 ); u.bt.pDb = &db->aDb[pOp->p1]; assert( u.bt.pDb->pBt!=0 ); if( pOp->opcode==OP_CreateTable ){ /* u.bt.flags = BTREE_INTKEY; */ - u.bt.flags = BTREE_LEAFDATA|BTREE_INTKEY; + u.bt.flags = BTREE_INTKEY; }else{ - u.bt.flags = BTREE_ZERODATA; + u.bt.flags = BTREE_BLOBKEY; } rc = sqlite3BtreeCreateTable(u.bt.pDb->pBt, &u.bt.pgno, u.bt.flags); pOut->u.i = u.bt.pgno; break; } @@ -63249,10 +65236,11 @@ void *t; /* Token identifying trigger */ #endif /* local variables moved into u.by */ u.by.pProgram = pOp->p4.pProgram; u.by.pRt = &aMem[pOp->p3]; + assert( memIsValid(u.by.pRt) ); assert( u.by.pProgram->nOp>0 ); /* If the p5 flag is clear, then recursive invocation of triggers is ** disabled for backwards compatibility (p5 is set if this sub-program ** is really a trigger, not a foreign key action, and the flag set @@ -63422,10 +65410,11 @@ for(u.ca.pFrame=p->pFrame; u.ca.pFrame->pParent; u.ca.pFrame=u.ca.pFrame->pParent); u.ca.pIn1 = &u.ca.pFrame->aMem[pOp->p1]; }else{ u.ca.pIn1 = &aMem[pOp->p1]; } + assert( memIsValid(u.ca.pIn1) ); sqlite3VdbeMemIntegerify(u.ca.pIn1); pIn2 = &aMem[pOp->p2]; sqlite3VdbeMemIntegerify(pIn2); if( u.ca.pIn1->u.iu.i){ u.ca.pIn1->u.i = pIn2->u.i; @@ -63508,11 +65497,13 @@ assert( u.cb.n>=0 ); u.cb.pRec = &aMem[pOp->p2]; u.cb.apVal = p->apArg; assert( u.cb.apVal || u.cb.n==0 ); for(u.cb.i=0; u.cb.ip4.pFunc; assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.cb.ctx.pMem = u.cb.pMem = &aMem[pOp->p3]; @@ -63528,11 +65519,11 @@ assert( pOp>p->aOp ); assert( pOp[-1].p4type==P4_COLLSEQ ); assert( pOp[-1].opcode==OP_CollSeq ); u.cb.ctx.pColl = pOp[-1].p4.pColl; } - (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); + (u.cb.ctx.pFunc->xStep)(&u.cb.ctx, u.cb.n, u.cb.apVal); /* IMP: R-24505-23230 */ if( u.cb.ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&u.cb.ctx.s)); rc = u.cb.ctx.isError; } sqlite3VdbeMemRelease(&u.cb.ctx.s); @@ -63641,15 +65632,15 @@ #ifndef SQLITE_OMIT_WAL u.cd.zFilename = sqlite3PagerFilename(u.cd.pPager); /* Do not allow a transition to journal_mode=WAL for a database - ** in temporary storage or if the VFS does not support xShmOpen. + ** in temporary storage or if the VFS does not support shared memory */ if( u.cd.eNew==PAGER_JOURNALMODE_WAL && (u.cd.zFilename[0]==0 /* Temp file */ - || !sqlite3PagerWalSupported(u.cd.pPager)) /* No xShmOpen support */ + || !sqlite3PagerWalSupported(u.cd.pPager)) /* No shared-memory support */ ){ u.cd.eNew = u.cd.eOld; } if( (u.cd.eNew!=u.cd.eOld) @@ -63807,15 +65798,11 @@ #if 0 /* local variables moved into u.cf */ VTable *pVTab; #endif /* local variables moved into u.cf */ u.cf.pVTab = pOp->p4.pVtab; rc = sqlite3VtabBegin(db, u.cf.pVTab); - if( u.cf.pVTab ){ - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cf.pVTab->pVtab->zErrMsg; - u.cf.pVTab->pVtab->zErrMsg = 0; - } + if( u.cf.pVTab ) importVtabErrMsg(p, u.cf.pVTab->pVtab); break; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -63863,13 +65850,11 @@ u.cg.pVtabCursor = 0; u.cg.pVtab = pOp->p4.pVtab->pVtab; u.cg.pModule = (sqlite3_module *)u.cg.pVtab->pModule; assert(u.cg.pVtab && u.cg.pModule); rc = u.cg.pModule->xOpen(u.cg.pVtab, &u.cg.pVtabCursor); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cg.pVtab->zErrMsg; - u.cg.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cg.pVtab); if( SQLITE_OK==rc ){ /* Initialize sqlite3_vtab_cursor base class */ u.cg.pVtabCursor->pVtab = u.cg.pVtab; /* Initialise vdbe cursor object */ @@ -63921,10 +65906,11 @@ #endif /* local variables moved into u.ch */ u.ch.pQuery = &aMem[pOp->p3]; u.ch.pArgc = &u.ch.pQuery[1]; u.ch.pCur = p->apCsr[pOp->p1]; + assert( memIsValid(u.ch.pQuery) ); REGISTER_TRACE(pOp->p3, u.ch.pQuery); assert( u.ch.pCur->pVtabCursor ); u.ch.pVtabCursor = u.ch.pCur->pVtabCursor; u.ch.pVtab = u.ch.pVtabCursor->pVtab; u.ch.pModule = u.ch.pVtab->pModule; @@ -63944,13 +65930,11 @@ } p->inVtabMethod = 1; rc = u.ch.pModule->xFilter(u.ch.pVtabCursor, u.ch.iQuery, pOp->p4.z, u.ch.nArg, u.ch.apArg); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ch.pVtab->zErrMsg; - u.ch.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ch.pVtab); if( rc==SQLITE_OK ){ u.ch.res = u.ch.pModule->xEof(u.ch.pVtabCursor); } if( u.ch.res ){ @@ -63980,10 +65964,11 @@ VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur->pVtabCursor ); assert( pOp->p3>0 && pOp->p3<=p->nMem ); u.ci.pDest = &aMem[pOp->p3]; + memAboutToChange(p, u.ci.pDest); if( pCur->nullRow ){ sqlite3VdbeMemSetNull(u.ci.pDest); break; } u.ci.pVtab = pCur->pVtabCursor->pVtab; @@ -63998,13 +65983,11 @@ */ sqlite3VdbeMemMove(&u.ci.sContext.s, u.ci.pDest); MemSetTypeFlag(&u.ci.sContext.s, MEM_Null); rc = u.ci.pModule->xColumn(pCur->pVtabCursor, &u.ci.sContext, pOp->p2); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ci.pVtab->zErrMsg; - u.ci.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ci.pVtab); if( u.ci.sContext.isError ){ rc = u.ci.sContext.isError; } /* Copy the result of the function to the P3 register. We @@ -64055,13 +66038,11 @@ ** some other method is next invoked on the save virtual table cursor. */ p->inVtabMethod = 1; rc = u.cj.pModule->xNext(u.cj.pCur->pVtabCursor); p->inVtabMethod = 0; - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cj.pVtab->zErrMsg; - u.cj.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cj.pVtab); if( rc==SQLITE_OK ){ u.cj.res = u.cj.pModule->xEof(u.cj.pCur->pVtabCursor); } if( !u.cj.res ){ @@ -64086,16 +66067,16 @@ #endif /* local variables moved into u.ck */ u.ck.pVtab = pOp->p4.pVtab->pVtab; u.ck.pName = &aMem[pOp->p1]; assert( u.ck.pVtab->pModule->xRename ); + assert( memIsValid(u.ck.pName) ); REGISTER_TRACE(pOp->p1, u.ck.pName); assert( u.ck.pName->flags & MEM_Str ); rc = u.ck.pVtab->pModule->xRename(u.ck.pVtab, u.ck.pName->z); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.ck.pVtab->zErrMsg; - u.ck.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.ck.pVtab); + p->expired = 0; break; } #endif @@ -64140,18 +66121,18 @@ assert( pOp->p4type==P4_VTAB ); if( ALWAYS(u.cl.pModule->xUpdate) ){ u.cl.apArg = p->apArg; u.cl.pX = &aMem[pOp->p3]; for(u.cl.i=0; u.cl.ixUpdate(u.cl.pVtab, u.cl.nArg, u.cl.apArg, &u.cl.rowid); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = u.cl.pVtab->zErrMsg; - u.cl.pVtab->zErrMsg = 0; + importVtabErrMsg(p, u.cl.pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( u.cl.nArg>1 && u.cl.apArg[0] && (u.cl.apArg[0]->flags&MEM_Null) ); db->lastRowid = u.cl.rowid; } p->nChange++; @@ -64168,10 +66149,36 @@ case OP_Pagecount: { /* out2-prerelease */ pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); break; } #endif + + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: MaxPgcnt P1 P2 P3 * * +** +** Try to set the maximum page count for database P1 to the value in P3. +** Do not let the maximum page count fall below the current page count and +** do not change the maximum page count value if P3==0. +** +** Store the maximum page count after the change in register P2. +*/ +case OP_MaxPgcnt: { /* out2-prerelease */ + unsigned int newMax; + Btree *pBt; + + pBt = db->aDb[pOp->p1].pBt; + newMax = 0; + if( pOp->p3 ){ + newMax = sqlite3BtreeLastPage(pBt); + if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; + } + pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + break; +} +#endif + #ifndef SQLITE_OMIT_TRACE /* Opcode: Trace * * * P4 * ** ** If tracing is enabled (by the sqlite3_trace()) interface, then @@ -64343,15 +66350,86 @@ typedef struct Incrblob Incrblob; struct Incrblob { int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ + int iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ }; + +/* +** This function is used by both blob_open() and blob_reopen(). It seeks +** the b-tree cursor associated with blob handle p to point to row iRow. +** If successful, SQLITE_OK is returned and subsequent calls to +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a value of type TEXT or BLOB in the column nominated when the +** blob handle was opened, then an error code is returned and *pzErr may +** be set to point to a buffer containing an error message. It is the +** responsibility of the caller to free the error message buffer using +** sqlite3DbFree(). +** +** If an error does occur, then the b-tree cursor is closed. All subsequent +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** immediately return SQLITE_ABORT. +*/ +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + /* Set the value of the SQL statements only variable to integer iRow. + ** This is done directly instead of using sqlite3_bind_int64() to avoid + ** triggering asserts related to mutexes. + */ + assert( v->aVar[0].flags&MEM_Int ); + v->aVar[0].u.i = iRow; + + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_ROW ){ + u32 type = v->apCsr[0]->aType[p->iCol]; + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = v->apCsr[0]->aOffset[p->iCol]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = v->apCsr[0]->pCursor; + sqlite3BtreeEnterCursor(p->pCsr); + sqlite3BtreeCacheOverflow(p->pCsr); + sqlite3BtreeLeaveCursor(p->pCsr); + } + } + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; +} + /* ** Open a blob handle. */ SQLITE_API int sqlite3_blob_open( sqlite3* db, /* The database connection */ @@ -64388,33 +66466,39 @@ /* One of the following two instructions is replaced by an OP_Noop. */ {OP_OpenRead, 0, 0, 0}, /* 3: Open cursor 0 for reading */ {OP_OpenWrite, 0, 0, 0}, /* 4: Open cursor 0 for read/write */ {OP_Variable, 1, 1, 1}, /* 5: Push the rowid to the stack */ - {OP_NotExists, 0, 9, 1}, /* 6: Seek the cursor */ + {OP_NotExists, 0, 10, 1}, /* 6: Seek the cursor */ {OP_Column, 0, 0, 1}, /* 7 */ {OP_ResultRow, 1, 0, 0}, /* 8 */ - {OP_Close, 0, 0, 0}, /* 9 */ - {OP_Halt, 0, 0, 0}, /* 10 */ + {OP_Goto, 0, 5, 0}, /* 9 */ + {OP_Close, 0, 0, 0}, /* 10 */ + {OP_Halt, 0, 0, 0}, /* 11 */ }; - Vdbe *v = 0; int rc = SQLITE_OK; char *zErr = 0; Table *pTab; - Parse *pParse; + Parse *pParse = 0; + Incrblob *pBlob = 0; + flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); + if( !pBlob ) goto blob_open_out; pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM; - goto blob_open_out; - } + if( !pParse ) goto blob_open_out; + do { memset(pParse, 0, sizeof(Parse)); pParse->db = db; + sqlite3DbFree(db, zErr); + zErr = 0; sqlite3BtreeEnterAll(db); pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); if( pTab && IsVirtual(pTab) ){ pTab = 0; @@ -64436,11 +66520,11 @@ sqlite3BtreeLeaveAll(db); goto blob_open_out; } /* Now search pTab for the exact column. */ - for(iCol=0; iCol < pTab->nCol; iCol++) { + for(iCol=0; iColnCol; iCol++) { if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ break; } } if( iCol==pTab->nCol ){ @@ -64490,15 +66574,18 @@ sqlite3BtreeLeaveAll(db); goto blob_open_out; } } - v = sqlite3VdbeCreate(db); - if( v ){ + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(db); + assert( pBlob->pStmt || db->mallocFailed ); + if( pBlob->pStmt ){ + Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3VdbeAddOpList(v, sizeof(openBlob)/sizeof(VdbeOpList), openBlob); - flags = !!flags; /* flags = (flags ? 1 : 0); */ + /* Configure the OP_Transaction */ sqlite3VdbeChangeP1(v, 0, iDb); sqlite3VdbeChangeP2(v, 0, flags); @@ -64537,69 +66624,29 @@ if( !db->mallocFailed ){ sqlite3VdbeMakeReady(v, 1, 1, 1, 0, 0, 0); } } + pBlob->flags = flags; + pBlob->iCol = iCol; + pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } - - sqlite3_bind_int64((sqlite3_stmt *)v, 1, iRow); - rc = sqlite3_step((sqlite3_stmt *)v); - if( rc!=SQLITE_ROW ){ - nAttempt++; - rc = sqlite3_finalize((sqlite3_stmt *)v); - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, sqlite3_errmsg(db)); - v = 0; - } - } while( nAttempt<5 && rc==SQLITE_SCHEMA ); - - if( rc==SQLITE_ROW ){ - /* The row-record has been opened successfully. Check that the - ** column in question contains text or a blob. If it contains - ** text, it is up to the caller to get the encoding right. - */ - Incrblob *pBlob; - u32 type = v->apCsr[0]->aType[iCol]; - - if( type<12 ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "cannot open value of type %s", - type==0?"null": type==7?"real": "integer" - ); - rc = SQLITE_ERROR; - goto blob_open_out; - } - pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( db->mallocFailed ){ - sqlite3DbFree(db, pBlob); - goto blob_open_out; - } - pBlob->flags = flags; - pBlob->pCsr = v->apCsr[0]->pCursor; - sqlite3BtreeEnterCursor(pBlob->pCsr); - sqlite3BtreeCacheOverflow(pBlob->pCsr); - sqlite3BtreeLeaveCursor(pBlob->pCsr); - pBlob->pStmt = (sqlite3_stmt *)v; - pBlob->iOffset = v->apCsr[0]->aOffset[iCol]; - pBlob->nByte = sqlite3VdbeSerialTypeLen(type); - pBlob->db = db; - *ppBlob = (sqlite3_blob *)pBlob; - rc = SQLITE_OK; - }else if( rc==SQLITE_OK ){ - sqlite3DbFree(db, zErr); - zErr = sqlite3MPrintf(db, "no such rowid: %lld", iRow); - rc = SQLITE_ERROR; - } + sqlite3_bind_int64(pBlob->pStmt, 1, iRow); + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); blob_open_out: - if( v && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(v); + if( rc==SQLITE_OK && db->mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, zErr); + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3StackFree(db, pParse); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -64648,11 +66695,11 @@ if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; sqlite3Error(db, SQLITE_ERROR, 0); - } else if( v==0 ){ + }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. */ rc = SQLITE_ABORT; }else{ @@ -64696,11 +66743,51 @@ ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; - return p ? p->nByte : 0; + return (p && p->pStmt) ? p->nByte : 0; +} + +/* +** Move an existing blob handle to point to a different row of the same +** database table. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); + } + + rc = sqlite3ApiExit(db, rc); + assert( rc==SQLITE_OK || p->pStmt==0 ); + sqlite3_mutex_leave(db->mutex); + return rc; } #endif /* #ifndef SQLITE_OMIT_INCRBLOB */ /************** End of vdbeblob.c ********************************************/ @@ -64887,15 +66974,14 @@ 0, /* xUnlock */ 0, /* xCheckReservedLock */ 0, /* xFileControl */ 0, /* xSectorSize */ 0, /* xDeviceCharacteristics */ - 0, /* xShmOpen */ + 0, /* xShmMap */ 0, /* xShmLock */ - 0, /* xShmMap */ 0, /* xShmBarrier */ - 0 /* xShmClose */ + 0 /* xShmUnmap */ }; /* ** Open a journal file. */ @@ -65172,15 +67258,14 @@ 0, /* xUnlock */ 0, /* xCheckReservedLock */ 0, /* xFileControl */ 0, /* xSectorSize */ 0, /* xDeviceCharacteristics */ - 0, /* xShmOpen */ + 0, /* xShmMap */ 0, /* xShmLock */ - 0, /* xShmMap */ 0, /* xShmBarrier */ - 0 /* xShmClose */ + 0 /* xShmUnlock */ }; /* ** Open a journal file. */ @@ -65198,12 +67283,11 @@ SQLITE_PRIVATE int sqlite3IsMemJournal(sqlite3_file *pJfd){ return pJfd->pMethods==&MemJournalMethods; } /* -** Return the number of bytes required to store a MemJournal that uses vfs -** pVfs to create the underlying on-disk files. +** Return the number of bytes required to store a MemJournal file descriptor. */ SQLITE_PRIVATE int sqlite3MemJournalSize(void){ return sizeof(MemJournal); } @@ -66602,30 +68686,37 @@ assert( pExpr->pTab && jpTab->nCol ); return pExpr->pTab->aCol[j].affinity; } return pExpr->affinity; } + +/* +** Set the explicit collating sequence for an expression to the +** collating sequence supplied in the second argument. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Expr *pExpr, CollSeq *pColl){ + if( pExpr && pColl ){ + pExpr->pColl = pColl; + pExpr->flags |= EP_ExpCollate; + } + return pExpr; +} /* ** Set the collating sequence for expression pExpr to be the collating ** sequence named by pToken. Return a pointer to the revised expression. ** The collating sequence is marked as "explicit" using the EP_ExpCollate ** flag. An explicit collating sequence will override implicit ** collating sequences. */ -SQLITE_PRIVATE Expr *sqlite3ExprSetColl(Parse *pParse, Expr *pExpr, Token *pCollName){ +SQLITE_PRIVATE Expr *sqlite3ExprSetCollByToken(Parse *pParse, Expr *pExpr, Token *pCollName){ char *zColl = 0; /* Dequoted name of collation sequence */ CollSeq *pColl; sqlite3 *db = pParse->db; zColl = sqlite3NameFromToken(db, pCollName); - if( pExpr && zColl ){ - pColl = sqlite3LocateCollSeq(pParse, zColl); - if( pColl ){ - pExpr->pColl = pColl; - pExpr->flags |= EP_ExpCollate; - } - } + pColl = sqlite3LocateCollSeq(pParse, zColl); + sqlite3ExprSetColl(pExpr, pColl); sqlite3DbFree(db, zColl); return pExpr; } /* @@ -67025,10 +69116,13 @@ Expr *pRight, /* Right operand */ const Token *pToken /* Argument token */ ){ Expr *p = sqlite3ExprAlloc(pParse->db, op, pToken, 1); sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + if( p ) { + sqlite3ExprCheckHeight(pParse, p->nHeight); + } return p; } /* ** Join two expressions using an AND operator. If either expression is @@ -67095,22 +69189,23 @@ assert( z[0]=='?' ); pExpr->iColumn = (ynVar)(++pParse->nVar); }else if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ - int i = atoi((char*)&z[1]); + i64 i; + int bOk = 0==sqlite3Atoi64(&z[1], &i, sqlite3Strlen30(&z[1]), SQLITE_UTF8); pExpr->iColumn = (ynVar)i; testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); - if( i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); } if( i>pParse->nVar ){ - pParse->nVar = i; + pParse->nVar = (int)i; } }else{ /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable ** number as the prior appearance of the same name, or if the name ** has never appeared before, reuse the same variable number @@ -68053,28 +70148,34 @@ if( eType==0 ){ /* Could not found an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. */ + double savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; eType = IN_INDEX_EPH; if( prNotFound ){ *prNotFound = rMayHaveNull = ++pParse->nMem; - }else if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ - eType = IN_INDEX_ROWID; + }else{ + testcase( pParse->nQueryLoop>(double)1 ); + pParse->nQueryLoop = (double)1; + if( pX->pLeft->iColumn<0 && !ExprHasAnyProperty(pX, EP_xIsSelect) ){ + eType = IN_INDEX_ROWID; + } } sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + pParse->nQueryLoop = savedNQueryLoop; }else{ pX->iTable = iTab; } return eType; } #endif /* -** Generate code for scalar subqueries used as an expression -** and IN operators. Examples: +** Generate code for scalar subqueries used as a subquery expression, EXISTS, +** or IN operators. Examples: ** ** (SELECT a FROM b) -- subquery ** EXISTS (SELECT a FROM b) -- EXISTS subquery ** x IN (4,5,11) -- IN operator with list on right-hand side ** x IN (SELECT a FROM b) -- IN operator with subquery on the right @@ -68130,17 +70231,27 @@ int mem = ++pParse->nMem; sqlite3VdbeAddOp1(v, OP_If, mem); testAddr = sqlite3VdbeAddOp2(v, OP_Integer, 1, mem); assert( testAddr>0 || pParse->db->mallocFailed ); } + +#ifndef SQLITE_OMIT_EXPLAIN + if( pParse->explain==2 ){ + char *zMsg = sqlite3MPrintf( + pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr?"":"CORRELATED ", + pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +#endif switch( pExpr->op ){ case TK_IN: { - char affinity; - KeyInfo keyInfo; - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; + char affinity; /* Affinity of the LHS of the IN */ + KeyInfo keyInfo; /* Keyinfo for the generated table */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ if( rMayHaveNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); } @@ -68159,10 +70270,11 @@ ** 'x' nor the SELECT... statement are columns, then numeric affinity ** is used. */ pExpr->iTable = pParse->nTab++; addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); + if( rMayHaveNull==0 ) sqlite3VdbeChangeP5(v, BTREE_UNORDERED); memset(&keyInfo, 0, sizeof(keyInfo)); keyInfo.nField = 1; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* Case 1: expr IN (SELECT ...) @@ -68175,19 +70287,20 @@ assert( !isRowid ); sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); dest.affinity = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); + pExpr->x.pSelect->iLimit = 0; if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ return 0; } pEList = pExpr->x.pSelect->pEList; if( ALWAYS(pEList!=0 && pEList->nExpr>0) ){ keyInfo.aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pEList->a[0].pExpr); } - }else if( pExpr->x.pList!=0 ){ + }else if( ALWAYS(pExpr->x.pList!=0) ){ /* Case 2: expr IN (exprlist) ** ** For each expression, build an index key from the evaluation and ** store it in the temporary table. If is a column, then use ** that columns affinity when building index keys. If is not @@ -68253,11 +70366,10 @@ ** value of this select in a memory cell and record the number ** of the memory cell in iColumn. If this is an EXISTS, write ** an integer 0 (not exists) or 1 (exists) into a memory cell ** and record that memory cell in iColumn. */ - static const Token one = { "1", 1 }; /* Token for literal value 1 */ Select *pSel; /* SELECT statement to encode */ SelectDest dest; /* How to deal with SELECt result */ testcase( pExpr->op==TK_EXISTS ); testcase( pExpr->op==TK_SELECT ); @@ -68274,11 +70386,13 @@ dest.eDest = SRT_Exists; sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iParm); VdbeComment((v, "Init EXISTS result")); } sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &one); + pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, + &sqlite3IntTokens[1]); + pSel->iLimit = 0; if( sqlite3Select(pParse, pSel, &dest) ){ return 0; } rReg = dest.iParm; ExprSetIrreducible(pExpr); @@ -68342,12 +70456,24 @@ /* Code the LHS, the from " IN (...)". */ sqlite3ExprCachePush(pParse); r1 = sqlite3GetTempReg(pParse); sqlite3ExprCode(pParse, pExpr->pLeft, r1); - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. + */ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } if( eType==IN_INDEX_ROWID ){ /* In this case, the RHS is the ROWID of table b-tree */ sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); @@ -68439,11 +70565,11 @@ */ static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ if( ALWAYS(z!=0) ){ double value; char *zV; - sqlite3AtoF(z, &value); + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ if( negateFlag ) value = -value; zV = dup8bytes(v, (char*)&value); sqlite3VdbeAddOp4(v, OP_Real, 0, iMem, 0, zV, P4_REAL); } @@ -68453,28 +70579,27 @@ /* ** Generate an instruction that will put the integer describe by ** text z[0..n-1] into register iMem. ** -** The z[] string will probably not be zero-terminated. But the -** z[n] character is guaranteed to be something that does not look -** like the continuation of the number. +** Expr.u.zToken is always UTF8 and zero-terminated. */ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ Vdbe *v = pParse->pVdbe; if( pExpr->flags & EP_IntValue ){ int i = pExpr->u.iValue; if( negFlag ) i = -i; sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); }else{ + int c; + i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - if( sqlite3FitsIn64Bits(z, negFlag) ){ - i64 value; + c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + if( c==0 || (c==2 && negFlag) ){ char *zV; - sqlite3Atoi64(z, &value); - if( negFlag ) value = -value; + if( negFlag ){ value = -value; } zV = dup8bytes(v, (char*)&value); sqlite3VdbeAddOp4(v, OP_Int64, 0, iMem, 0, zV, P4_INT64); }else{ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); @@ -68755,77 +70880,10 @@ } return 0; } #endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ -/* -** If the last instruction coded is an ephemeral copy of any of -** the registers in the nReg registers beginning with iReg, then -** convert the last instruction from OP_SCopy to OP_Copy. -*/ -SQLITE_PRIVATE void sqlite3ExprHardCopy(Parse *pParse, int iReg, int nReg){ - VdbeOp *pOp; - Vdbe *v; - - assert( pParse->db->mallocFailed==0 ); - v = pParse->pVdbe; - assert( v!=0 ); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - if( pOp->opcode==OP_SCopy && pOp->p1>=iReg && pOp->p1opcode = OP_Copy; - } -} - -/* -** Generate code to store the value of the iAlias-th alias in register -** target. The first time this is called, pExpr is evaluated to compute -** the value of the alias. The value is stored in an auxiliary register -** and the number of that register is returned. On subsequent calls, -** the register number is returned without generating any code. -** -** Note that in order for this to work, code must be generated in the -** same order that it is executed. -** -** Aliases are numbered starting with 1. So iAlias is in the range -** of 1 to pParse->nAlias inclusive. -** -** pParse->aAlias[iAlias-1] records the register number where the value -** of the iAlias-th alias is stored. If zero, that means that the -** alias has not yet been computed. -*/ -static int codeAlias(Parse *pParse, int iAlias, Expr *pExpr, int target){ -#if 0 - sqlite3 *db = pParse->db; - int iReg; - if( pParse->nAliasAllocnAlias ){ - pParse->aAlias = sqlite3DbReallocOrFree(db, pParse->aAlias, - sizeof(pParse->aAlias[0])*pParse->nAlias ); - testcase( db->mallocFailed && pParse->nAliasAlloc>0 ); - if( db->mallocFailed ) return 0; - memset(&pParse->aAlias[pParse->nAliasAlloc], 0, - (pParse->nAlias-pParse->nAliasAlloc)*sizeof(pParse->aAlias[0])); - pParse->nAliasAlloc = pParse->nAlias; - } - assert( iAlias>0 && iAlias<=pParse->nAlias ); - iReg = pParse->aAlias[iAlias-1]; - if( iReg==0 ){ - if( pParse->iCacheLevel>0 ){ - iReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - }else{ - iReg = ++pParse->nMem; - sqlite3ExprCode(pParse, pExpr, iReg); - pParse->aAlias[iAlias-1] = iReg; - } - } - return iReg; -#else - UNUSED_PARAMETER(iAlias); - return sqlite3ExprCodeTarget(pParse, pExpr, target); -#endif -} - /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". ** Return the register where results are stored. ** @@ -68930,11 +70988,11 @@ case TK_REGISTER: { inReg = pExpr->iTable; break; } case TK_AS: { - inReg = codeAlias(pParse, pExpr->iTable, pExpr->pLeft, target); + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ @@ -69362,10 +71420,15 @@ testcase( regFree1==0 ); cacheX.op = TK_REGISTER; opCompare.op = TK_EQ; opCompare.pLeft = &cacheX; pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; } for(i=0; i0 && target<=pParse->nMem ); - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + if( pExpr && pExpr->op==TK_REGISTER ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); + }else{ + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + } } return target; } /* @@ -69605,13 +71672,26 @@ /* ** Preevaluate constant subexpressions within pExpr and store the ** results in registers. Modify pExpr so that the constant subexpresions ** are TK_REGISTER opcodes that refer to the precomputed values. +** +** This routine is a no-op if the jump to the cookie-check code has +** already occur. Since the cookie-check jump is generated prior to +** any other serious processing, this check ensures that there is no +** way to accidently bypass the constant initializations. +** +** This routine is also a no-op if the SQLITE_FactorOutConst optimization +** is disabled via the sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS) +** interface. This allows test logic to verify that the same answer is +** obtained for queries regardless of whether or not constants are +** precomputed into registers or if they are inserted in-line. */ SQLITE_PRIVATE void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; + if( pParse->cookieGoto ) return; + if( (pParse->db->flags & SQLITE_FactorOutConst)!=0 ) return; w.xExprCallback = evalConstExpr; w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } @@ -69631,23 +71711,18 @@ ){ struct ExprList_item *pItem; int i, n; assert( pList!=0 ); assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ n = pList->nExpr; for(pItem=pList->a, i=0; iiAlias ){ - int iReg = codeAlias(pParse, pItem->iAlias, pItem->pExpr, target+i); - Vdbe *v = sqlite3GetVdbe(pParse); - if( iReg!=target+i ){ - sqlite3VdbeAddOp2(v, OP_SCopy, iReg, target+i); - } - }else{ - sqlite3ExprCode(pParse, pItem->pExpr, target+i); - } - if( doHardCopy && !pParse->db->mallocFailed ){ - sqlite3ExprHardCopy(pParse, target, n); + Expr *pExpr = pItem->pExpr; + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + sqlite3VdbeAddOp2(pParse->pVdbe, doHardCopy ? OP_Copy : OP_SCopy, + inReg, target+i); } } return n; } @@ -70625,10 +72700,15 @@ if( pTrig->pSchema==pTempSchema ){ zWhere = whereOrName(db, zWhere, pTrig->zName); } } } + if( zWhere ){ + char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); + sqlite3DbFree(pParse->db, zWhere); + zWhere = zNew; + } return zWhere; } /* ** Generate code to drop and reload the internal representation of table @@ -71081,11 +73161,10 @@ */ pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); if( !pNew ) goto exit_begin_add_column; pParse->pNewTable = pNew; pNew->nRef = 1; - pNew->dbMem = pTab->dbMem; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); @@ -71233,11 +73312,12 @@ int iIdxCur; /* Cursor open on index being analyzed */ Vdbe *v; /* The virtual machine being built up */ int i; /* Loop counter */ int topOfLoop; /* The top of the loop */ int endOfLoop; /* The end of the loop */ - int addr; /* The address of an instruction */ + int addr = 0; /* The address of an instruction */ + int jZeroRows = 0; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regSampleno = iMem++; /* Register containing next sample number */ int regCol = iMem++; /* Content of a column analyzed table */ @@ -71252,12 +73332,19 @@ int regLast = iMem++; /* Index of last sample to record */ int regFirst = iMem++; /* Index of first sample to record */ #endif v = sqlite3GetVdbe(pParse); - if( v==0 || NEVER(pTab==0) || pTab->pIndex==0 ){ - /* Do no analysis for tables that have no indices */ + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( memcmp(pTab->zName, "sqlite_", 7)==0 ){ + /* Do not gather statistics on system tables */ return; } assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDb>=0 ); @@ -71270,10 +73357,11 @@ /* Establish a read-lock on the table at the shared-cache level. */ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol = pIdx->nColumn; KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); if( iMem+1+(nCol*2)>pParse->nMem ){ @@ -71284,14 +73372,11 @@ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, (char *)pKey, P4_KEYINFO_HANDOFF); VdbeComment((v, "%s", pIdx->zName)); - /* Populate the registers containing the table and index names. */ - if( pTab->pIndex==pIdx ){ - sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); - } + /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); #ifdef SQLITE_ENABLE_STAT2 /* If this iteration of the loop is generating code to analyze the @@ -71422,12 +73507,14 @@ ** ** If K==0 then no entry is made into the sqlite_stat1 table. ** If K>0 then it is always the case the D>0 so division by zero ** is never possible. */ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno); + if( jZeroRows==0 ){ + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + } for(i=0; ipIndex==0 ){ + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb); + VdbeComment((v, "%s", pTab->zName)); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno); + sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); + }else{ + assert( jZeroRows>0 ); + addr = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, jZeroRows); + } + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + if( pParse->nMemnMem = regRec; + if( jZeroRows ){ sqlite3VdbeJumpHere(v, addr); } } /* ** Generate code that will cause the most recent index analysis to -** be laoded into internal hash tables where is can be used. +** be loaded into internal hash tables where is can be used. */ static void loadAnalysis(Parse *pParse, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); if( v ){ sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); @@ -71573,37 +73682,50 @@ /* ** This callback is invoked once for each index when reading the ** sqlite_stat1 table. ** -** argv[0] = name of the index -** argv[1] = results of analysis - on integer for each column +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. */ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; - int i, c; + Table *pTable; + int i, c, n; unsigned int v; const char *z; - assert( argc==2 ); + assert( argc==3 ); UNUSED_PARAMETER2(NotUsed, argc); - if( argv==0 || argv[0]==0 || argv[1]==0 ){ + if( argv==0 || argv[0]==0 || argv[2]==0 ){ return 0; } - pIndex = sqlite3FindIndex(pInfo->db, argv[0], pInfo->zDatabase); - if( pIndex==0 ){ + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ return 0; } - z = argv[1]; - for(i=0; *z && i<=pIndex->nColumn; i++){ + if( argv[1] ){ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + }else{ + pIndex = 0; + } + n = pIndex ? pIndex->nColumn : 0; + z = argv[2]; + for(i=0; *z && i<=n; i++){ v = 0; while( (c=z[0])>='0' && c<='9' ){ v = v*10 + c - '0'; z++; } + if( i==0 ) pTable->nRowEst = v; + if( pIndex==0 ) break; pIndex->aiRowEst[i] = v; if( *z==' ' ) z++; } return 0; } @@ -71610,25 +73732,24 @@ /* ** If the Index.aSample variable is not NULL, delete the aSample[] array ** and its contents. */ -SQLITE_PRIVATE void sqlite3DeleteIndexSamples(Index *pIdx){ +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #ifdef SQLITE_ENABLE_STAT2 if( pIdx->aSample ){ int j; - sqlite3 *dbMem = pIdx->pTable->dbMem; for(j=0; jaSample[j]; if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(pIdx->pTable->dbMem, p->u.z); + sqlite3DbFree(db, p->u.z); } } - sqlite3DbFree(dbMem, pIdx->aSample); - pIdx->aSample = 0; + sqlite3DbFree(db, pIdx->aSample); } #else + UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); #endif } /* @@ -71663,11 +73784,12 @@ /* Clear any prior statistics */ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); - sqlite3DeleteIndexSamples(pIdx); + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; } /* Check to make sure the sqlite_stat1 table exists */ sInfo.db = db; sInfo.zDatabase = db->aDb[iDb].zName; @@ -71675,11 +73797,11 @@ return SQLITE_ERROR; } /* Load new statistics out of the sqlite_stat1 table */ zSql = sqlite3MPrintf(db, - "SELECT idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); sqlite3DbFree(db, zSql); @@ -71707,22 +73829,21 @@ while( sqlite3_step(pStmt)==SQLITE_ROW ){ char *zIndex = (char *)sqlite3_column_text(pStmt, 0); Index *pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase); if( pIdx ){ int iSample = sqlite3_column_int(pStmt, 1); - sqlite3 *dbMem = pIdx->pTable->dbMem; - assert( dbMem==db || dbMem==0 ); if( iSample=0 ){ int eType = sqlite3_column_type(pStmt, 2); if( pIdx->aSample==0 ){ static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES; - pIdx->aSample = (IndexSample *)sqlite3DbMallocZero(dbMem, sz); + pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz); if( pIdx->aSample==0 ){ db->mallocFailed = 1; break; } + memset(pIdx->aSample, 0, sz); } assert( pIdx->aSample ); { IndexSample *pSample = &pIdx->aSample[iSample]; @@ -71741,14 +73862,12 @@ } pSample->nByte = (u8)n; if( n < 1){ pSample->u.z = 0; }else{ - pSample->u.z = sqlite3DbMallocRaw(dbMem, n); - if( pSample->u.z ){ - memcpy(pSample->u.z, z, n); - }else{ + pSample->u.z = sqlite3DbStrNDup(0, z, n); + if( pSample->u.z==0 ){ db->mallocFailed = 1; break; } } } @@ -71895,13 +74014,12 @@ /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may ** or may not be initialised. */ - rc = sqlite3BtreeFactory(db, zFile, 0, SQLITE_DEFAULT_CACHE_SIZE, - db->openFlags | SQLITE_OPEN_MAIN_DB, - &aNew->pBt); + rc = sqlite3BtreeOpen(zFile, db, &aNew->pBt, 0, + db->openFlags | SQLITE_OPEN_MAIN_DB); db->nDb++; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; zErrDyn = sqlite3MPrintf(db, "database is already attached"); }else if( rc==SQLITE_OK ){ @@ -72138,11 +74256,12 @@ 0, /* pNext */ detachFunc, /* xFunc */ 0, /* xStep */ 0, /* xFinalize */ "sqlite_detach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); } /* @@ -72159,11 +74278,12 @@ 0, /* pNext */ attachFunc, /* xFunc */ 0, /* xStep */ 0, /* xFinalize */ "sqlite_attach", /* zName */ - 0 /* pHash */ + 0, /* pHash */ + 0 /* pDestructor */ }; codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); } #endif /* SQLITE_OMIT_ATTACH */ @@ -72907,37 +75027,18 @@ } /* ** Reclaim the memory used by an index */ -static void freeIndex(Index *p){ - sqlite3 *db = p->pTable->dbMem; +static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE - sqlite3DeleteIndexSamples(p); + sqlite3DeleteIndexSamples(db, p); #endif sqlite3DbFree(db, p->zColAff); sqlite3DbFree(db, p); } -/* -** Remove the given index from the index hash table, and free -** its memory structures. -** -** The index is removed from the database hash tables but -** it is not unlinked from the Table that it indexes. -** Unlinking from the Table must be done by the calling function. -*/ -static void sqlite3DeleteIndex(Index *p){ - Index *pOld; - const char *zName = p->zName; - - pOld = sqlite3HashInsert(&p->pSchema->idxHash, zName, - sqlite3Strlen30(zName), 0); - assert( pOld==0 || pOld==p ); - freeIndex(p); -} - /* ** For the index called zIdxName which is found in the database iDb, ** unlike that index from its Table then remove the index from ** the index hash table and free all memory structures associated ** with the index. @@ -72960,11 +75061,11 @@ while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } if( ALWAYS(p && p->pNext==pIndex) ){ p->pNext = pIndex->pNext; } } - freeIndex(pIndex); + freeIndex(db, pIndex); } db->flags |= SQLITE_InternChanges; } /* @@ -73031,17 +75132,16 @@ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ db->flags &= ~SQLITE_InternChanges; } /* -** Clear the column names from a table or view. +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). */ -static void sqliteResetColumnNames(Table *pTable){ +static void sqliteDeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; - sqlite3 *db = pTable->dbMem; - testcase( db==0 ); assert( pTable!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); @@ -73049,12 +75149,10 @@ sqlite3DbFree(db, pCol->zType); sqlite3DbFree(db, pCol->zColl); } sqlite3DbFree(db, pTable->aCol); } - pTable->aCol = 0; - pTable->nCol = 0; } /* ** Remove the memory data structures associated with the given ** Table. No changes are made to disk by this routine. @@ -73062,46 +75160,48 @@ ** This routine just deletes the data structure. It does not unlink ** the table data structure from the hash table. But it does destroy ** memory structures of the indices and foreign keys associated with ** the table. */ -SQLITE_PRIVATE void sqlite3DeleteTable(Table *pTable){ +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; - sqlite3 *db; - if( pTable==0 ) return; - db = pTable->dbMem; - testcase( db==0 ); + assert( !pTable || pTable->nRef>0 ); /* Do not delete the table until the reference count reaches zero. */ - pTable->nRef--; - if( pTable->nRef>0 ){ - return; - } - assert( pTable->nRef==0 ); - - /* Delete all indices associated with this table - */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + + /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema ); - sqlite3DeleteIndex(pIndex); + if( !db || db->pnBytesFreed==0 ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + ); + assert( pOld==pIndex || pOld==0 ); + } + freeIndex(db, pIndex); } /* Delete any foreign keys attached to this table. */ - sqlite3FkDelete(pTable); + sqlite3FkDelete(db, pTable); /* Delete the Table structure itself. */ - sqliteResetColumnNames(pTable); + sqliteDeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); sqlite3SelectDelete(db, pTable->pSelect); #ifndef SQLITE_OMIT_CHECK sqlite3ExprDelete(db, pTable->pCheck); #endif - sqlite3VtabClear(pTable); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3VtabClear(db, pTable); +#endif sqlite3DbFree(db, pTable); } /* ** Unlink the given table from the hash tables and the delete the @@ -73116,11 +75216,11 @@ assert( zTabName ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, sqlite3Strlen30(zTabName),0); - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } /* ** Given a token, return a string that consists of the text of that @@ -73308,12 +75408,13 @@ ** set to the index of the database that the table or view is to be ** created in. */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) return; - if( !OMIT_TEMPDB && isTemp && iDb>1 ){ - /* If creating a temp table, the name may not be qualified */ + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); return; } if( !OMIT_TEMPDB && isTemp ) iDb = 1; @@ -73357,21 +75458,22 @@ ** to an sqlite3_declare_vtab() call. In that case only the column names ** and types will be used, so there is no need to test for namespace ** collisions. */ if( !IN_DECLARE_VTAB ){ + char *zDb = db->aDb[iDb].zName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } - pTable = sqlite3FindTable(db, zName, db->aDb[iDb].zName); + pTable = sqlite3FindTable(db, zName, zDb); if( pTable ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "table %T already exists", pName); } goto begin_table_error; } - if( sqlite3FindIndex(db, zName, 0)!=0 && (iDb==0 || !db->init.busy) ){ + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); goto begin_table_error; } } @@ -73384,11 +75486,11 @@ } pTable->zName = zName; pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->dbMem = 0; + pTable->nRowEst = 1000000; assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; /* If this is the magic sqlite_sequence table used by autoincrement, ** then record a pointer to this table in the main database structure @@ -73936,11 +76038,11 @@ zSep = "\n "; zSep2 = ",\n "; zEnd = "\n)"; } n += 35 + 6*p->nCol; - zStmt = sqlite3Malloc( n ); + zStmt = sqlite3DbMallocRaw(0, n); if( zStmt==0 ){ db->mallocFailed = 1; return 0; } sqlite3_snprintf(n, zStmt, "CREATE TABLE "); @@ -74117,11 +76219,11 @@ assert( p->aCol==0 ); p->nCol = pSelTab->nCol; p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); } } /* Compute the complete text of the CREATE statement */ if( pSelect ){ @@ -74231,16 +76333,14 @@ sqlite3SelectDelete(db, pSelect); return; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; - if( p==0 ){ + if( p==0 || pParse->nErr ){ sqlite3SelectDelete(db, pSelect); return; } - assert( pParse->nErr==0 ); /* If sqlite3StartTable return non-NULL then - ** there could not have been an error */ sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); if( sqlite3FixInit(&sFix, pParse, iDb, "view", pName) && sqlite3FixSelect(&sFix, pSelect) ){ @@ -74361,11 +76461,11 @@ assert( pTable->aCol==0 ); pTable->nCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; - sqlite3DeleteTable(pSelTab); + sqlite3DeleteTable(db, pSelTab); pTable->pSchema->flags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; } @@ -74386,11 +76486,13 @@ HashElem *i; if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); if( pTab->pSelect ){ - sqliteResetColumnNames(pTab); + sqliteDeleteColumnNames(db, pTab); + pTab->aCol = 0; + pTab->nCol = 0; } } DbClearProperty(db, idx, DB_UnresetViews); } #else @@ -75352,11 +77454,12 @@ */ if( pTblName ){ sqlite3RefillIndex(pParse, pIndex, iMem); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 0, 0, - sqlite3MPrintf(db, "name='%q'", pIndex->zName), P4_DYNAMIC); + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), + P4_DYNAMIC); sqlite3VdbeAddOp1(v, OP_Expire, 0); } } /* When adding an index to the list of indices for a table, make @@ -75383,11 +77486,11 @@ } /* Clean up before exiting */ exit_create_index: if( pIndex ){ - sqlite3_free(pIndex->zColAff); + sqlite3DbFree(db, pIndex->zColAff); sqlite3DbFree(db, pIndex); } sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); @@ -75413,18 +77516,18 @@ ** are based on typical values found in actual indices. */ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ unsigned *a = pIdx->aiRowEst; int i; + unsigned n; assert( a!=0 ); - a[0] = 1000000; - for(i=pIdx->nColumn; i>=5; i--){ - a[i] = 5; - } - while( i>=1 ){ - a[i] = 11 - i; - i--; + a[0] = pIdx->pTable->nRowEst; + if( a[0]<10 ) a[0] = 10; + n = 10; + for(i=1; i<=pIdx->nColumn; i++){ + a[i] = n; + if( n>5 ) n--; } if( pIdx->onError!=OE_None ){ a[pIdx->nColumn] = 1; } } @@ -75480,11 +77583,11 @@ /* Generate code to remove the index and from the master table */ v = sqlite3GetVdbe(pParse); if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q", + "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName ); if( sqlite3FindTable(db, "sqlite_stat1", db->aDb[iDb].zName) ){ sqlite3NestedParse(pParse, @@ -75762,11 +77865,11 @@ for(pItem=pList->a, i=0; inSrc; i++, pItem++){ sqlite3DbFree(db, pItem->zDatabase); sqlite3DbFree(db, pItem->zName); sqlite3DbFree(db, pItem->zAlias); sqlite3DbFree(db, pItem->zIndex); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(db, pItem->pTab); sqlite3SelectDelete(db, pItem->pSelect); sqlite3ExprDelete(db, pItem->pOn); sqlite3IdListDelete(db, pItem->pUsing); } sqlite3DbFree(db, pList); @@ -75972,11 +78075,11 @@ SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_TEMP_DB; - rc = sqlite3BtreeFactory(db, 0, 0, SQLITE_DEFAULT_CACHE_SIZE, flags, &pBt); + rc = sqlite3BtreeOpen(0, db, &pBt, 0, flags); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "unable to open a temporary database " "file for storing temporary tables"); pParse->rc = rc; return 1; @@ -76629,11 +78732,11 @@ ** If the SQLITE_PreferBuiltin flag is set, then search the built-in ** functions even if a prior app-defined function was found. And give ** priority to built-in functions. ** ** Except, if createFlag is true, that means that we are trying to - ** install a new function. Whatever FuncDef structure is returned will + ** install a new function. Whatever FuncDef structure is returned it will ** have fields overwritten with new information appropriate for the ** new function. But the FuncDefs for built-in functions are read-only. ** So we must not search for built-ins when creating a new function. */ if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ @@ -76693,12 +78796,11 @@ } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - assert( pTab->dbMem==0 ); - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(0, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; pSchema->flags &= ~DB_SchemaLoaded; @@ -76711,11 +78813,11 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ Schema * p; if( pBt ){ p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaFree); }else{ - p = (Schema *)sqlite3MallocZero(sizeof(Schema)); + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); } if( !p ){ db->mallocFailed = 1; }else if ( 0==p->file_format ){ sqlite3HashInit(&p->tblHash); @@ -76752,11 +78854,11 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ struct SrcList_item *pItem = pSrc->a; Table *pTab; assert( pItem && pSrc->nSrc==1 ); pTab = sqlite3LocateTable(pParse, 0, pItem->zName, pItem->zDatabase); - sqlite3DeleteTable(pItem->pTab); + sqlite3DeleteTable(pParse->db, pItem->pTab); pItem->pTab = pTab; if( pTab ){ pTab->nRef++; } if( sqlite3IndexedByLookup(pParse, pItem) ){ @@ -77646,11 +79748,11 @@ zBuf = sqlite3_mprintf("%.*f",n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; } - sqlite3AtoF(zBuf, &r); + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); sqlite3_free(zBuf); } sqlite3_result_double(context, r); } #endif @@ -78149,12 +80251,14 @@ sqlite3_value **argv ){ const char *zOptName; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_used() C interface. */ + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); } } #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ @@ -78171,12 +80275,13 @@ sqlite3_value **argv ){ int n; assert( argc==1 ); UNUSED_PARAMETER(argc); - /* IMP: R-xxxx This function is an SQL wrapper around the - ** sqlite3_compileoption_get() C interface. */ + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ n = sqlite3_value_int(argv[0]); sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); } #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ @@ -78371,18 +80476,18 @@ nOut += nRep - nPattern; testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ sqlite3_result_error_toobig(context); - sqlite3DbFree(db, zOut); + sqlite3_free(zOut); return; } zOld = zOut; zOut = sqlite3_realloc(zOut, (int)nOut); if( zOut==0 ){ sqlite3_result_error_nomem(context); - sqlite3DbFree(db, zOld); + sqlite3_free(zOld); return; } memcpy(&zOut[j], zRep, nRep); j += nRep; i += nPattern-1; @@ -78739,11 +80844,11 @@ pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); if( pAccum ){ sqlite3 *db = sqlite3_context_db_handle(context); int firstTerm = pAccum->useMalloc==0; - pAccum->useMalloc = 1; + pAccum->useMalloc = 2; pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; if( !firstTerm ){ if( argc==2 ){ zSep = (char*)sqlite3_value_text(argv[1]); nSep = sqlite3_value_bytes(argv[1]); @@ -78808,14 +80913,14 @@ if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; }else{ pInfo = (struct compareInfo*)&likeInfoNorm; } - sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0); + sqlite3CreateFunc(db, "like", 2, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_ANY, pInfo, likeFunc, 0, 0, 0); sqlite3CreateFunc(db, "glob", 2, SQLITE_ANY, - (struct compareInfo*)&globInfo, likeFunc, 0,0); + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); setLikeOptFlag(db, "like", caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); } @@ -78895,14 +81000,14 @@ FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), /* FUNCTION(coalesce, -1, 0, 0, ifnullFunc ), */ - {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0}, + {-1,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"coalesce",0,0}, FUNCTION(hex, 1, 0, 0, hexFunc ), /* FUNCTION(ifnull, 2, 0, 0, ifnullFunc ), */ - {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0}, + {2,SQLITE_UTF8,SQLITE_FUNC_COALESCE,0,0,ifnullFunc,0,0,"ifnull",0,0}, FUNCTION(random, 0, 0, 0, randomFunc ), FUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), @@ -78925,11 +81030,11 @@ #endif AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0}, + {0,SQLITE_UTF8,SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), @@ -79336,11 +81441,11 @@ KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx); sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); for(i=0; iaiColumn[i]; - pCol = &pIdx->pTable->aCol[iCol]; + pCol = &pTab->aCol[iCol]; + if( pTab->iPKey==iCol ) iCol = -1; pLeft->iTable = regData+iCol+1; pLeft->affinity = pCol->affinity; pLeft->pColl = sqlite3LocateCollSeq(pParse, pCol->zColl); }else{ pLeft->iTable = regData; @@ -80017,15 +82123,11 @@ 0, 0, 0, 0, 0, 0 ); pWhere = 0; } - /* In the current implementation, pTab->dbMem==0 for all tables except - ** for temporary tables used to describe subqueries. And temporary - ** tables do not have foreign key constraints. Hence, pTab->dbMem - ** should always be 0 there. - */ + /* Disable lookaside memory allocation */ enableLookaside = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ @@ -80111,41 +82213,43 @@ /* ** Free all memory associated with foreign key definitions attached to ** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash ** hash table. */ -SQLITE_PRIVATE void sqlite3FkDelete(Table *pTab){ +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ /* Remove the FK from the fkeyHash hash table. */ - if( pFKey->pPrevTo ){ - pFKey->pPrevTo->pNextTo = pFKey->pNextTo; - }else{ - void *data = (void *)pFKey->pNextTo; - const char *z = (data ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), data); - } - if( pFKey->pNextTo ){ - pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; - } - - /* Delete any triggers created to implement actions for this FK. */ -#ifndef SQLITE_OMIT_TRIGGER - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[0]); - fkTriggerDelete(pTab->dbMem, pFKey->apTrigger[1]); -#endif + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } + } /* EV: R-30323-21917 Each foreign key constraint in SQLite is ** classified as either immediate or deferred. */ assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif pNext = pFKey->pNextFrom; - sqlite3DbFree(pTab->dbMem, pFKey); + sqlite3DbFree(db, pFKey); } } #endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ /************** End of fkey.c ************************************************/ @@ -80216,11 +82320,11 @@ ** up. */ int n; Table *pTab = pIdx->pTable; sqlite3 *db = sqlite3VdbeDb(v); - pIdx->zColAff = (char *)sqlite3Malloc(pIdx->nColumn+2); + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+2); if( !pIdx->zColAff ){ db->mallocFailed = 1; return 0; } for(n=0; nnColumn; n++){ @@ -80258,11 +82362,11 @@ if( !pTab->zColAff ){ char *zColAff; int i; sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3Malloc(pTab->nCol+1); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); if( !zColAff ){ db->mallocFailed = 1; return; } @@ -81369,10 +83473,11 @@ sqlite3ExprIfTrue(pParse, pTab->pCheck, allOk, SQLITE_JUMPIFNULL); onError = overrideError!=OE_Default ? overrideError : OE_Abort; if( onError==OE_Ignore ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, ignoreDest); }else{ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ sqlite3HaltConstraint(pParse, onError, 0, 0); } sqlite3VdbeResolveLabel(v, allOk); } #endif /* !defined(SQLITE_OMIT_CHECK) */ @@ -82338,10 +84443,31 @@ int (*extended_result_codes)(sqlite3*,int); int (*limit)(sqlite3*,int,int); sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); const char *(*sql)(sqlite3_stmt*); int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*),void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); }; /* ** The following macros redefine the API routines so that they are ** redirected throught the global sqlite3_api structure. @@ -82517,10 +84643,31 @@ #define sqlite3_extended_result_codes sqlite3_api->extended_result_codes #define sqlite3_limit sqlite3_api->limit #define sqlite3_next_stmt sqlite3_api->next_stmt #define sqlite3_sql sqlite3_api->sql #define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; #define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; @@ -82834,10 +84981,50 @@ sqlite3_extended_result_codes, sqlite3_limit, sqlite3_next_stmt, sqlite3_sql, sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif }; /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a @@ -82883,32 +85070,28 @@ } handle = sqlite3OsDlOpen(pVfs, zFile); if( handle==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "unable to open shared library [%s]", zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } } return SQLITE_ERROR; } xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) sqlite3OsDlSym(pVfs, handle, zProc); if( xInit==0 ){ if( pzErrMsg ){ - zErrmsg = sqlite3StackAllocZero(db, nMsg); + *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zProc,zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); - *pzErrMsg = sqlite3DbStrDup(0, zErrmsg); - sqlite3StackFree(db, zErrmsg); } sqlite3OsDlClose(pVfs, handle); } return SQLITE_ERROR; }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ @@ -83153,11 +85336,11 @@ static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 4}; static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 2}; int i, n; if( sqlite3Isdigit(*z) ){ - return (u8)atoi(z); + return (u8)sqlite3Atoi(z); } n = sqlite3Strlen30(z); for(i=0; i=0&&i<=2)?i:0); } #endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS @@ -83290,10 +85473,11 @@ { "short_column_names", SQLITE_ShortColNames }, { "count_changes", SQLITE_CountRows }, { "empty_result_callbacks", SQLITE_NullCallback }, { "legacy_file_format", SQLITE_LegacyFileFmt }, { "fullfsync", SQLITE_FullFSync }, + { "checkpoint_fullfsync", SQLITE_CkptFullFSync }, { "reverse_unordered_selects", SQLITE_ReverseOrder }, #ifndef SQLITE_OMIT_AUTOMATIC_INDEX { "automatic_index", SQLITE_AutoIndex }, #endif #ifdef SQLITE_DEBUG @@ -83501,11 +85685,11 @@ addr = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize); sqlite3VdbeChangeP1(v, addr, iDb); sqlite3VdbeChangeP1(v, addr+1, iDb); sqlite3VdbeChangeP1(v, addr+6, SQLITE_DEFAULT_CACHE_SIZE); }else{ - int size = atoi(zRight); + int size = sqlite3Atoi(zRight); if( size<0 ) size = -size; sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3VdbeAddOp2(v, OP_Integer, size, 1); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, 1); pDb->pSchema->cache_size = size; @@ -83530,39 +85714,17 @@ returnSingleInt(pParse, "page_size", size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ - db->nextPagesize = atoi(zRight); + db->nextPagesize = sqlite3Atoi(zRight); if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ db->mallocFailed = 1; } } }else - /* - ** PRAGMA [database.]max_page_count - ** PRAGMA [database.]max_page_count=N - ** - ** The first form reports the current setting for the - ** maximum number of pages in the database file. The - ** second form attempts to change this setting. Both - ** forms return the current setting. - */ - if( sqlite3StrICmp(zLeft,"max_page_count")==0 ){ - Btree *pBt = pDb->pBt; - int newMax = 0; - assert( pBt!=0 ); - if( zRight ){ - newMax = atoi(zRight); - } - if( ALWAYS(pBt) ){ - newMax = sqlite3BtreeMaxPageCount(pBt, newMax); - } - returnSingleInt(pParse, "max_page_count", newMax); - }else - /* ** PRAGMA [database.]secure_delete ** PRAGMA [database.]secure_delete=ON/OFF ** ** The first form reports the current setting for the @@ -83585,23 +85747,37 @@ b = sqlite3BtreeSecureDelete(pBt, b); returnSingleInt(pParse, "secure_delete", b); }else /* + ** PRAGMA [database.]max_page_count + ** PRAGMA [database.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** ** PRAGMA [database.]page_count ** ** Return the number of pages in the specified database. */ - if( sqlite3StrICmp(zLeft,"page_count")==0 ){ + if( sqlite3StrICmp(zLeft,"page_count")==0 + || sqlite3StrICmp(zLeft,"max_page_count")==0 + ){ int iReg; if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3CodeVerifySchema(pParse, iDb); iReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + if( zLeft[0]=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, sqlite3Atoi(zRight)); + } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "page_count", SQLITE_STATIC); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); }else /* ** PRAGMA [database.]locking_mode ** PRAGMA [database.]locking_mode = (normal|exclusive) @@ -83705,11 +85881,11 @@ */ if( sqlite3StrICmp(zLeft,"journal_size_limit")==0 ){ Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit); + sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); returnSingleInt(pParse, "journal_size_limit", iLimit); }else @@ -83819,11 +85995,11 @@ if( sqlite3StrICmp(zLeft,"cache_size")==0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; if( !zRight ){ returnSingleInt(pParse, "cache_size", pDb->pSchema->cache_size); }else{ - int size = atoi(zRight); + int size = sqlite3Atoi(zRight); if( size<0 ) size = -size; pDb->pSchema->cache_size = size; sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); } }else @@ -83883,11 +86059,11 @@ ){ invalidateTempStorage(pParse); } sqlite3_free(sqlite3_temp_directory); if( zRight[0] ){ - sqlite3_temp_directory = sqlite3DbStrDup(0, zRight); + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); }else{ sqlite3_temp_directory = 0; } #endif /* SQLITE_OMIT_WSD */ } @@ -84212,11 +86388,11 @@ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "integrity_check", SQLITE_STATIC); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - mxErr = atoi(zRight); + sqlite3GetInt32(zRight, &mxErr); if( mxErr<=0 ){ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; } } sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ @@ -84469,11 +86645,11 @@ { OP_Integer, 0, 1, 0}, /* 1 */ { OP_SetCookie, 0, 0, 1}, /* 2 */ }; int addr = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie); sqlite3VdbeChangeP1(v, addr, iDb); - sqlite3VdbeChangeP1(v, addr+1, atoi(zRight)); + sqlite3VdbeChangeP1(v, addr+1, sqlite3Atoi(zRight)); sqlite3VdbeChangeP1(v, addr+2, iDb); sqlite3VdbeChangeP2(v, addr+2, iCookie); }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -84530,12 +86706,11 @@ ** after accumulating N frames in the log. Or query for the current value ** of N. */ if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ if( zRight ){ - int nAuto = atoi(zRight); - sqlite3_wal_autocheckpoint(db, nAuto); + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } returnSingleInt(pParse, "wal_autocheckpoint", db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); }else @@ -84621,11 +86796,12 @@ ** setting changed. */ #ifndef SQLITE_OMIT_PAGER_PRAGMAS if( db->autoCommit ){ sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level, - (db->flags&SQLITE_FullFSync)!=0); + (db->flags&SQLITE_FullFSync)!=0, + (db->flags&SQLITE_CkptFullFSync)!=0); } #endif pragma_out: sqlite3DbFree(db, zLeft); sqlite3DbFree(db, zRight); @@ -84713,11 +86889,11 @@ sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ assert( db->init.busy ); db->init.iDb = iDb; - db->init.newTnum = atoi(argv[1]); + db->init.newTnum = sqlite3Atoi(argv[1]); db->init.orphanTrigger = 0; TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = 0; @@ -85262,17 +87438,17 @@ #ifndef SQLITE_OMIT_EXPLAIN if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ static const char * const azColName[] = { "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "order", "from", "detail" + "selectid", "order", "from", "detail" }; int iFirst, mx; if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 3); + sqlite3VdbeSetNumCols(pParse->pVdbe, 4); iFirst = 8; - mx = 11; + mx = 12; }else{ sqlite3VdbeSetNumCols(pParse->pVdbe, 8); iFirst = 0; mx = 8; } @@ -85304,11 +87480,10 @@ /* Delete any TriggerPrg structures allocated while parsing this statement. */ while( pParse->pTriggerPrg ){ TriggerPrg *pT = pParse->pTriggerPrg; pParse->pTriggerPrg = pT->pNext; - sqlite3VdbeProgramDelete(db, pT->pProgram, 0); sqlite3DbFree(db, pT); } end_prepare: @@ -85939,11 +88114,10 @@ addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); sqlite3VdbeJumpHere(v, addr2); - pSelect->iLimit = 0; } } /* ** Add code to implement the OFFSET @@ -85988,15 +88162,17 @@ sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); sqlite3ReleaseTempReg(pParse, r1); } +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate an error message when a SELECT is used within a subexpression ** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error occurs in multiple -** places. +** column. We do this in a subroutine because the error used to occur +** in multiple places. (The error only occurs in one place now, but we +** retain the subroutine to minimize code disruption.) */ static int checkForMultiColumnSelectError( Parse *pParse, /* Parse context. */ SelectDest *pDest, /* Destination of SELECT results */ int nExpr /* Number of result columns returned by SELECT */ @@ -86008,10 +88184,11 @@ return 1; }else{ return 0; } } +#endif /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** @@ -86087,14 +88264,10 @@ if( pOrderBy==0 ){ codeOffset(v, p, iContinue); } } - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - return; - } - switch( eDest ){ /* In this mode, write each query result to the key of the temporary ** table iParm. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -86219,15 +88392,15 @@ break; } #endif } - /* Jump to the end of the loop if the LIMIT is reached. + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. */ - if( p->iLimit ){ - assert( pOrderBy==0 ); /* If there is an ORDER BY, the call to - ** pushOntoSorter() would have cleared p->iLimit */ + if( pOrderBy==0 && p->iLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); } } /* @@ -86270,10 +88443,92 @@ } } return pInfo; } +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Name of the connection operator, used for error messages. +*/ +static const char *selectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; + } + return z; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} + +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of one of the two forms: +** +** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" +** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" +** +** where iSub1 and iSub2 are the integers passed as the corresponding +** function parameters, and op is the text representation of the parameter +** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, +** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is +** false, or the second form if it is true. +*/ +static void explainComposite( + Parse *pParse, /* Parse context */ + int op, /* One of TK_UNION, TK_EXCEPT etc. */ + int iSub1, /* Subquery id 1 */ + int iSub2, /* Subquery id 2 */ + int bUseTmp /* True if a temp table was used */ +){ + assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); + if( pParse->explain==2 ){ + Vdbe *v = pParse->pVdbe; + char *zMsg = sqlite3MPrintf( + pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, + bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) + ); + sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + } +} + +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b + +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainComposite(v,w,x,y,z) +# define explainSetInteger(y,z) +#endif /* ** If the inner loop was generated using a non-null pOrderBy argument, ** then the results were placed in a sorter. After the loop is terminated ** we need to run the sorter and output the results. The following @@ -86358,14 +88613,10 @@ } } sqlite3ReleaseTempReg(pParse, regRow); sqlite3ReleaseTempReg(pParse, regRowid); - /* LIMIT has been implemented by the pushOntoSorter() routine. - */ - assert( p->iLimit==0 ); - /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); sqlite3VdbeResolveLabel(v, addrBreak); @@ -86621,26 +88872,10 @@ } } generateColumnTypes(pParse, pTabList, pEList); } -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** Name of the connection operator, used for error messages. -*/ -static const char *selectOpName(int id){ - char *z; - switch( id ){ - case TK_ALL: z = "UNION ALL"; break; - case TK_INTERSECT: z = "INTERSECT"; break; - case TK_EXCEPT: z = "EXCEPT"; break; - default: z = "UNION"; break; - } - return z; -} -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - /* ** Given a an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate ** column names for a table that would hold the expression list. ** @@ -86795,20 +89030,20 @@ pTab = sqlite3DbMallocZero(db, sizeof(Table) ); if( pTab==0 ){ return 0; } /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled, so we might as well hard-code pTab->dbMem to NULL. */ + ** is disabled */ assert( db->lookaside.bEnabled==0 ); - pTab->dbMem = 0; pTab->nRef = 1; pTab->zName = 0; + pTab->nRowEst = 1000000; selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab->nCol, pTab->aCol, pSelect); pTab->iPKey = -1; if( db->mallocFailed ){ - sqlite3DeleteTable(pTab); + sqlite3DeleteTable(db, pTab); return 0; } return pTab; } @@ -86870,10 +89105,12 @@ if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + }else{ + if( p->nSelectRow > (double)n ) p->nSelectRow = (double)n; } }else{ sqlite3ExprCode(pParse, p->pLimit, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeComment((v, "LIMIT counter")); @@ -86970,10 +89207,14 @@ Select *pPrior; /* Another SELECT immediately to our left */ Vdbe *v; /* Generate code to this VDBE */ SelectDest dest; /* Alternative data destination */ Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ @@ -87001,10 +89242,11 @@ /* Create the destination temporary table if necessary */ if( dest.eDest==SRT_EphemTab ){ assert( p->pEList ); sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iParm, p->pEList->nExpr); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); dest.eDest = SRT_Table; } /* Make sure all SELECTs in the statement have the same number of elements ** in their result sets. @@ -87026,13 +89268,15 @@ /* Generate code for the left and right SELECT statements. */ switch( p->op ){ case TK_ALL: { int addr = 0; + int nLimit; assert( !pPrior->pLimit ); pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &dest); p->pLimit = 0; p->pOffset = 0; if( rc ){ goto multi_select_end; @@ -87042,14 +89286,22 @@ p->iOffset = pPrior->iOffset; if( p->iLimit ){ addr = sqlite3VdbeAddOp1(v, OP_IfZero, p->iLimit); VdbeComment((v, "Jump ahead if LIMIT reached")); } + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; + p->nSelectRow += pPrior->nSelectRow; + if( pPrior->pLimit + && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) + && p->nSelectRow > (double)nLimit + ){ + p->nSelectRow = (double)nLimit; + } if( addr ){ sqlite3VdbeJumpHere(v, addr); } break; } @@ -87089,10 +89341,11 @@ /* Code the SELECT statements to our left */ assert( !pPrior->pOrderBy ); sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &uniondest); if( rc ){ goto multi_select_end; } @@ -87108,18 +89361,20 @@ pLimit = p->pLimit; p->pLimit = 0; pOffset = p->pOffset; p->pOffset = 0; uniondest.eDest = op; + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &uniondest); testcase( rc!=SQLITE_OK ); /* Query flattening in sqlite3Select() might refill p->pOrderBy. ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ sqlite3ExprListDelete(db, p->pOrderBy); pDelete = p->pPrior; p->pPrior = pPrior; p->pOrderBy = 0; + if( p->op==TK_UNION ) p->nSelectRow += pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; p->iLimit = 0; p->iOffset = 0; @@ -87173,10 +89428,11 @@ assert( p->pEList ); /* Code the SELECTs to our left into temporary table "tab1". */ sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + explainSetInteger(iSub1, pParse->iNextSelectId); rc = sqlite3Select(pParse, pPrior, &intersectdest); if( rc ){ goto multi_select_end; } @@ -87189,14 +89445,16 @@ pLimit = p->pLimit; p->pLimit = 0; pOffset = p->pOffset; p->pOffset = 0; intersectdest.iParm = tab2; + explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &intersectdest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; sqlite3ExprDelete(db, p->pLimit); p->pLimit = pLimit; p->pOffset = pOffset; /* Generate code to take the intersection of the two temporary @@ -87224,10 +89482,12 @@ sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); break; } } + + explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); /* Compute collating sequences used by ** temporary tables needed to implement the compound select. ** Attach the KeyInfo structure to all temporary tables. ** @@ -87296,11 +89556,11 @@ ** be sent. ** ** regReturn is the number of the register holding the subroutine ** return address. ** -** If regPrev>0 then it is a the first register in a vector that +** If regPrev>0 then it is the first register in a vector that ** records the previous output. mem[regPrev] is a flag that is false ** if there has been no previous output. If regPrev>0 then code is ** generated to suppress duplicates. pKeyInfo is used for comparing ** keys. ** @@ -87568,10 +89828,14 @@ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ sqlite3 *db; /* Database connection */ ExprList *pOrderBy; /* The ORDER BY clause */ int nOrderBy; /* Number of terms in the ORDER BY clause */ int *aPermute; /* Mapping from ORDER BY terms to result set columns */ +#ifndef SQLITE_OMIT_EXPLAIN + int iSub1; /* EQP id of left-hand query */ + int iSub2; /* EQP id of right-hand query */ +#endif assert( p->pOrderBy!=0 ); assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ db = pParse->db; v = pParse->pVdbe; @@ -87679,11 +89943,10 @@ } /* Separate the left and the right query from one another */ p->pPrior = 0; - pPrior->pRightmost = 0; sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); if( pPrior->pPrior==0 ){ sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); } @@ -87722,10 +89985,11 @@ /* Generate a coroutine to evaluate the SELECT statement to the ** left of the compound operator - the "A" select. */ VdbeNoopComment((v, "Begin coroutine for left SELECT")); pPrior->iLimit = regLimitA; + explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofA); sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); VdbeNoopComment((v, "End coroutine for left SELECT")); @@ -87736,10 +90000,11 @@ VdbeNoopComment((v, "Begin coroutine for right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; p->iOffset = 0; + explainSetInteger(iSub2, pParse->iNextSelectId); sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; sqlite3VdbeAddOp2(v, OP_Integer, 1, regEofB); sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); @@ -87772,17 +90037,19 @@ }else{ addrEofA = sqlite3VdbeAddOp2(v, OP_If, regEofB, labelEnd); sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); sqlite3VdbeAddOp1(v, OP_Yield, regAddrB); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrEofA); + p->nSelectRow += pPrior->nSelectRow; } /* Generate a subroutine to run when the results from select B ** are exhausted and only data in select A remains. */ if( op==TK_INTERSECT ){ addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; }else{ VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_If, regEofA, labelEnd); sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp1(v, OP_Yield, regAddrA); @@ -87866,10 +90133,11 @@ } p->pPrior = pPrior; /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ + explainComposite(pParse, p->op, iSub1, iSub2, 0); return SQLITE_OK; } #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) @@ -87993,16 +90261,17 @@ ** (1) The subquery and the outer query do not both use aggregates. ** ** (2) The subquery is not an aggregate or the outer query is not a join. ** ** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strenghtened by ticket #3300) +** (Originally ticket #306. Strengthened by ticket #3300) ** -** (4) The subquery is not DISTINCT or the outer query is not a join. +** (4) The subquery is not DISTINCT. ** -** (5) The subquery is not DISTINCT or the outer query does not use -** aggregates. +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. ** ** (6) The subquery does not use aggregates or the outer query is not ** DISTINCT. ** ** (7) The subquery has a FROM clause. @@ -88018,13 +90287,13 @@ ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** ** (**) Not implemented. Subsumed into restriction (3). Was previously ** a separate restriction deriving from ticket #350. ** -** (13) The subquery and outer query do not both use LIMIT +** (13) The subquery and outer query do not both use LIMIT. ** -** (14) The subquery does not use OFFSET +** (14) The subquery does not use OFFSET. ** ** (15) The outer query is not part of a compound select or the ** subquery does not have a LIMIT clause. ** (See ticket #2339 and ticket [02a8e81d44]). ** @@ -88111,13 +90380,13 @@ if( pSub->pOffset ) return 0; /* Restriction (14) */ if( p->pRightmost && pSub->pLimit ){ return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( ((pSub->selFlags & SF_Distinct)!=0 || pSub->pLimit) - && (pSrc->nSrc>1 || isAgg) ){ /* Restrictions (4)(5)(8)(9) */ - return 0; + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ } if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ return 0; /* Restriction (6) */ } if( p->pOrderBy && pSub->pOrderBy ){ @@ -88593,16 +90862,16 @@ assert( pSel!=0 ); assert( pFrom->pTab==0 ); sqlite3WalkSelect(pWalker, pSel); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; - pTab->dbMem = db->lookaside.bEnabled ? db : 0; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_subquery_%p_", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; + pTab->nRowEst = 1000000; pTab->tabFlags |= TF_Ephemeral; #endif }else{ /* An ordinary table or view name in the FROM clause */ assert( pFrom->pTab==0 ); @@ -88963,11 +91232,11 @@ ExprList *pList = pF->pExpr->x.pList; assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); - sqlite3ExprCodeExprList(pParse, pList, regAgg, 0); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 1); }else{ nArg = 0; regAgg = 0; } if( pF->iDistinct>=0 ){ @@ -89091,10 +91360,15 @@ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ + +#ifndef SQLITE_OMIT_EXPLAIN + int iRestoreSelectId = pParse->iSelectId; + pParse->iSelectId = pParse->iNextSelectId++; +#endif db = pParse->db; if( p==0 || db->mallocFailed || pParse->nErr ){ return 1; } @@ -89122,10 +91396,19 @@ /* Begin generating code. */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto select_end; + + /* If writing to memory or generating a set + ** only a single column may be output. + */ +#ifndef SQLITE_OMIT_SUBQUERY + if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ + goto select_end; + } +#endif /* Generate code for all sub-queries in the FROM clause */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ @@ -89154,12 +91437,14 @@ } i = -1; }else{ sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); assert( pItem->isPopulated==0 ); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); pItem->isPopulated = 1; + pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; } if( /*pParse->nErr ||*/ db->mallocFailed ){ goto select_end; } pParse->nHeight -= sqlite3SelectExprHeight(p); @@ -89189,23 +91474,16 @@ pRight = pLoop; } mxSelect = db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT]; if( mxSelect && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - return 1; + goto select_end; } } - return multiSelect(pParse, p, pDest); - } -#endif - - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, pEList->nExpr) ){ - goto select_end; + rc = multiSelect(pParse, p, pDest); + explainSetInteger(pParse->iSelectId, iRestoreSelectId); + return rc; } #endif /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. ** GROUP BY might use an index, DISTINCT never does. @@ -89213,11 +91491,10 @@ assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; p->selFlags &= ~SF_Distinct; - isDistinct = 0; } /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY ** will cause elements to come out in the correct order. This is @@ -89256,21 +91533,23 @@ } /* Set the limiter. */ iEnd = sqlite3VdbeMakeLabel(v); + p->nSelectRow = (double)LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); /* Open a virtual index to use for the distinct set. */ - if( isDistinct ){ + if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ distinct = -1; } /* Aggregate and non-aggregate queries are handled differently */ @@ -89278,10 +91557,11 @@ /* This case is for non-aggregate queries ** Begin the database scan */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); if( pWInfo==0 ) goto select_end; + if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -89322,10 +91602,13 @@ pItem->iAlias = 0; } for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ pItem->iAlias = 0; } + if( p->nSelectRow>(double)100 ) p->nSelectRow = (double)100; + }else{ + p->nSelectRow = (double)1; } /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(v); @@ -89418,10 +91701,13 @@ int regBase; int regRecord; int nCol; int nGroupBy; + explainTempTable(pParse, + isDistinct && !(p->selFlags&SF_Distinct)?"DISTINCT":"GROUP BY"); + groupBySort = 1; nGroupBy = pGroupBy->nExpr; nCol = nGroupBy + 1; j = nGroupBy+1; for(i=0; i=0 ){ + explainTempTable(pParse, "DISTINCT"); + } /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ if( pOrderBy ){ + explainTempTable(pParse, "ORDER BY"); generateSortTail(pParse, p, v, pEList->nExpr, pDest); } /* Jump here to skip this query */ @@ -89699,10 +91990,11 @@ /* Control jumps to here if an error is encountered above, or upon ** successful coding of the SELECT. */ select_end: + explainSetInteger(pParse->iSelectId, iRestoreSelectId); /* Identify column names if results of the SELECT are to be output. */ if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ generateColumnNames(pParse, pTabList, pEList); @@ -90807,10 +93099,11 @@ SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ Parse *pSubParse; /* Parse context for sub-vdbe */ int iEndTrigger = 0; /* Label to jump to if WHEN is false */ assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); /* Allocate the TriggerPrg and SubProgram objects. To ensure that they ** are freed if an error occurs, link them into the Parse.pTriggerPrg ** list of the top-level Parse object sooner rather than later. */ pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); @@ -90817,11 +93110,11 @@ if( !pPrg ) return 0; pPrg->pNext = pTop->pTriggerPrg; pTop->pTriggerPrg = pPrg; pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); if( !pProgram ) return 0; - pProgram->nRef = 1; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); pPrg->pTrigger = pTrigger; pPrg->orconf = orconf; pPrg->aColmask[0] = 0xffffffff; pPrg->aColmask[1] = 0xffffffff; @@ -90951,22 +93244,23 @@ assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + sqlite3VdbeAddOp3(v, OP_Program, reg, ignoreJump, ++pParse->nMem); - pPrg->pProgram->nRef++; sqlite3VdbeChangeP4(v, -1, (const char *)pPrg->pProgram, P4_SUBPROGRAM); VdbeComment( (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); /* Set the P5 operand of the OP_Program instruction to non-zero if ** recursive invocation of this trigger program is disallowed. Recursive ** invocation is disallowed if (a) the sub-program is really a trigger, ** not a foreign key action, and (b) the flag to enable recursive triggers ** is clear. */ - sqlite3VdbeChangeP5(v, (u8)(p->zName && !(pParse->db->flags&SQLITE_RecTriggers))); + sqlite3VdbeChangeP5(v, (u8)bRecursive); } } /* ** This is called to code the required FOR EACH ROW triggers for an operation @@ -91111,11 +93405,11 @@ ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -sqlite************************************************************************* +************************************************************************* ** This file contains C code routines that are called by the parser ** to handle UPDATE statements. */ #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -91740,10 +94034,11 @@ ** be stored. */ assert( v ); ephemTab = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, pTab->nCol+1+(pRowid!=0)); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); /* fill the ephemeral table */ sqlite3SelectDestInit(&dest, SRT_Table, ephemTab); sqlite3Select(pParse, pSelect, &dest); @@ -91878,10 +94173,14 @@ int nDb; /* Number of attached databases */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); return SQLITE_ERROR; + } + if( db->activeVdbeCnt>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; } /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and ** disable CHECK and foreign key constraints. */ @@ -92330,18 +94629,18 @@ ** connection db is decremented immediately (which may lead to the ** structure being xDisconnected and free). Any other VTable structures ** in the list are moved to the sqlite3.pDisconnect list of the associated ** database connection. */ -SQLITE_PRIVATE void sqlite3VtabClear(Table *p){ - vtabDisconnectAll(0, p); +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); if( p->azModuleArg ){ int i; for(i=0; inModuleArg; i++){ - sqlite3DbFree(p->dbMem, p->azModuleArg[i]); + sqlite3DbFree(db, p->azModuleArg[i]); } - sqlite3DbFree(p->dbMem, p->azModuleArg); + sqlite3DbFree(db, p->azModuleArg); } } /* ** Add a new module argument to pTable->azModuleArg[]. @@ -92480,11 +94779,11 @@ sqlite3DbFree(db, zStmt); v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - zWhere = sqlite3MPrintf(db, "name='%q'", pTab->zName); + zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); sqlite3VdbeAddOp4(v, OP_ParseSchema, iDb, 1, 0, zWhere, P4_DYNAMIC); sqlite3VdbeAddOp4(v, OP_VCreate, iDb, 0, 0, pTab->zName, sqlite3Strlen30(pTab->zName) + 1); } @@ -92502,11 +94801,10 @@ if( pOld ){ db->mallocFailed = 1; assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ return; } - pSchema->db = pParse->db; pParse->pNewTable = 0; } } /* @@ -92576,11 +94874,11 @@ if( SQLITE_OK!=rc ){ if( zErr==0 ){ *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); }else { *pzErr = sqlite3MPrintf(db, "%s", zErr); - sqlite3DbFree(db, zErr); + sqlite3_free(zErr); } sqlite3DbFree(db, pVTable); }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ @@ -92782,20 +95080,20 @@ pParse->pNewTable->nCol = 0; pParse->pNewTable->aCol = 0; } db->pVTab = 0; }else{ - sqlite3Error(db, SQLITE_ERROR, zErr); + sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } pParse->declareVtab = 0; if( pParse->pVdbe ){ sqlite3VdbeFinalize(pParse->pVdbe); } - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); sqlite3StackFree(db, pParse); } assert( (rc&0xff)==rc ); rc = sqlite3ApiExit(db, rc); @@ -92878,12 +95176,12 @@ int (*x)(sqlite3_vtab *); sqlite3_vtab *pVtab = aVTrans[i]->pVtab; if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ rc = x(pVtab); sqlite3DbFree(db, *pzErrmsg); - *pzErrmsg = pVtab->zErrMsg; - pVtab->zErrMsg = 0; + *pzErrmsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); } } db->aVTrans = aVTrans; return rc; } @@ -93244,11 +95542,10 @@ ** cost of pursuing that strategy. */ struct WhereCost { WherePlan plan; /* The lookup strategy */ double rCost; /* Overall cost of pursuing this search strategy */ - double nRow; /* Estimated number of output rows */ Bitmask used; /* Bitmask of cursors used by this plan */ }; /* ** Bitmasks for the operators that indices are able to exploit. An @@ -93287,14 +95584,15 @@ #define WHERE_COLUMN_EQ 0x00010000 /* x=EXPR or x IN (...) or x IS NULL */ #define WHERE_COLUMN_RANGE 0x00020000 /* xEXPR */ #define WHERE_COLUMN_IN 0x00040000 /* x IN (...) */ #define WHERE_COLUMN_NULL 0x00080000 /* x IS NULL */ #define WHERE_INDEXED 0x000f0000 /* Anything that uses an index */ -#define WHERE_NOT_FULLSCAN 0x000f3000 /* Does not do a full table scan */ +#define WHERE_NOT_FULLSCAN 0x100f3000 /* Does not do a full table scan */ #define WHERE_IN_ABLE 0x000f1000 /* Able to support an IN operator */ #define WHERE_TOP_LIMIT 0x00100000 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00300000 /* Both x>EXPR and xa[] array. */ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ WhereTerm *pTerm; int idx; + testcase( wtFlags & TERM_VIRTUAL ); /* EV: R-00211-15100 */ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pParse->db; pWC->a = sqlite3DbMallocRaw(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ @@ -93524,10 +95823,17 @@ /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are ** "=", "<", ">", "<=", ">=", and "IN". +** +** IMPLEMENTATION-OF: R-59926-26393 To be usable by an index a term must be +** of one of the following forms: column = expression column > expression +** column >= expression column < expression column <= expression +** expression = column expression > column expression >= column +** expression < column expression <= column column IN +** (expression-list) column IN (subquery) column IS NULL */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LEdb; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ @@ -93706,54 +96011,42 @@ /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must ** be the name of an indexed column with TEXT affinity. */ return 0; } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pColl = sqlite3ExprCollSeq(pParse, pLeft); - if( pColl==0 ) return 0; /* Happens when LHS has an undefined collation */ - if( (pColl->type!=SQLITE_COLL_BINARY || *pnoCase) && - (pColl->type!=SQLITE_COLL_NOCASE || !*pnoCase) ){ - /* IMP: R-09003-32046 For the GLOB operator, the column must use the - ** default BINARY collating sequence. - ** IMP: R-41408-28306 For the LIKE operator, if case_sensitive_like mode - ** is enabled then the column must use the default BINARY collating - ** sequence, or if case_sensitive_like mode is disabled then the column - ** must use the built-in NOCASE collating sequence. - */ - return 0; - } pRight = pList->a[0].pExpr; op = pRight->op; if( op==TK_REGISTER ){ op = pRight->op2; } if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; - pVal = sqlite3VdbeGetValue(pReprepare, pRight->iColumn, SQLITE_AFF_NONE); + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetValue(pReprepare, iCol, SQLITE_AFF_NONE); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ z = (char *)sqlite3_value_text(pVal); } - sqlite3VdbeSetVarmask(pParse->pVdbe, pRight->iColumn); + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); /* IMP: R-23257-02778 */ assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ z = pRight->u.zToken; } if( z ){ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; } - if( cnt!=0 && c!=0 && 255!=(u8)z[cnt-1] ){ + if( cnt!=0 && 255!=(u8)z[cnt-1] ){ Expr *pPrefix; - *pisComplete = z[cnt]==wc[0] && z[cnt+1]==0; + *pisComplete = c==wc[0] && z[cnt+1]==0; pPrefix = sqlite3Expr(db, TK_STRING, z); if( pPrefix ) pPrefix->u.zToken[cnt] = 0; *ppPrefix = pPrefix; if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; - sqlite3VdbeSetVarmask(v, pRight->iColumn); + sqlite3VdbeSetVarmask(v, pRight->iColumn); /* IMP: R-23257-02778 */ if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() @@ -94082,10 +96375,12 @@ } /* At this point, okToChngToIN is true if original pTerm satisfies ** case 1. In that case, construct a new virtual term that is ** pTerm converted into an IN operator. + ** + ** EV: R-00211-15100 */ if( okToChngToIN ){ Expr *pDup; /* A transient duplicate expression */ ExprList *pList = 0; /* The RHS of the IN operator */ Expr *pLeft = 0; /* The LHS of the IN operator */ @@ -94298,10 +96593,11 @@ Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ Expr *pNewExpr1; Expr *pNewExpr2; int idxNew1; int idxNew2; + CollSeq *pColl; /* Collating sequence to use */ pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); if( !db->mallocFailed ){ u8 c, *pC; /* Last character before the first wildcard */ @@ -94312,21 +96608,27 @@ ** wildcard. But if we increment '@', that will push it into the ** alphabetic range where case conversions will mess up the ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ - if( c=='A'-1 ) isComplete = 0; + if( c=='A'-1 ) isComplete = 0; /* EV: R-64339-08207 */ + c = sqlite3UpperToLower[c]; } *pC = c + 1; } - pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprDup(db,pLeft,0),pStr1,0); + pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, noCase ? "NOCASE" : "BINARY",0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr1, 0); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); - pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprDup(db,pLeft,0),pStr2,0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprSetColl(sqlite3ExprDup(db,pLeft,0), pColl), + pStr2, 0); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ @@ -94604,11 +96906,12 @@ /* ** Required because bestIndex() is called by bestOrClauseIndex() */ static void bestIndex( - Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*); + Parse*, WhereClause*, struct SrcList_item*, + Bitmask, Bitmask, ExprList*, WhereCost*); /* ** This routine attempts to find an scanning strategy that can be used ** to optimize an 'OR' expression that is part of a WHERE clause. ** @@ -94617,22 +96920,24 @@ */ static void bestOrClauseIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ #ifndef SQLITE_OMIT_OR_OPTIMIZATION const int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur); /* Bitmask for pSrc */ WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm]; /* End of pWC->a[] */ WhereTerm *pTerm; /* A single term of the WHERE clause */ - /* No OR-clause optimization allowed if the NOT INDEXED clause is used */ - if( pSrc->notIndexed ){ + /* No OR-clause optimization allowed if the INDEXED BY or NOT INDEXED clauses + ** are used */ + if( pSrc->notIndexed || pSrc->pIndex!=0 ){ return; } /* Search the WHERE clause terms for a usable WO_OR term. */ for(pTerm=pWC->a; pTerma), (pTerm - pWC->a) )); if( pOrTerm->eOperator==WO_AND ){ WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc; - bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, pAndWC, pSrc, notReady, notValid, 0, &sTermCost); }else if( pOrTerm->leftCursor==iCur ){ WhereClause tempWC; tempWC.pParse = pWC->pParse; tempWC.pMaskSet = pWC->pMaskSet; tempWC.op = TK_AND; tempWC.a = pOrTerm; tempWC.nTerm = 1; - bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost); + bestIndex(pParse, &tempWC, pSrc, notReady, notValid, 0, &sTermCost); }else{ continue; } rTotal += sTermCost.rCost; - nRow += sTermCost.nRow; + nRow += sTermCost.plan.nRow; used |= sTermCost.used; if( rTotal>=pCost->rCost ) break; } /* If there is an ORDER BY clause, increase the scan cost to account @@ -94685,12 +96990,12 @@ ** less than the current cost stored in pCost, replace the contents ** of pCost. */ WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow)); if( rTotalrCost ){ pCost->rCost = rTotal; - pCost->nRow = nRow; pCost->used = used; + pCost->plan.nRow = nRow; pCost->plan.wsFlags = flags; pCost->plan.u.pTerm = pTerm; } } } @@ -94754,11 +97059,11 @@ return; } assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; - nTableRow = pTable->pIndex ? pTable->pIndex->aiRowEst[0] : 1000000; + nTableRow = pTable->nRowEst; logN = estLog(nTableRow); costTempIdx = 2*logN*(nTableRow/pParse->nQueryLoop + 1); if( costTempIdx>=pCost->rCost ){ /* The cost of creating the transient table would be greater than ** doing the full table scan */ @@ -94770,11 +97075,11 @@ for(pTerm=pWC->a; pTermrCost, costTempIdx)); pCost->rCost = costTempIdx; - pCost->nRow = logN + 1; + pCost->plan.nRow = logN + 1; pCost->plan.wsFlags = WHERE_TEMP_INDEX; pCost->used = pTerm->prereqRight; break; } } @@ -95074,11 +97379,11 @@ sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); }else{ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } - sqlite3DbFree(pParse->db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; for(i=0; inConstraint; i++){ if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ sqlite3ErrorMsg(pParse, @@ -95108,11 +97413,12 @@ */ static void bestVirtualIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for index */ + Bitmask notValid, /* Cursors not valid for any purpose */ ExprList *pOrderBy, /* The order by clause */ WhereCost *pCost, /* Lowest cost query plan */ sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */ ){ Table *pTab = pSrc->pTab; @@ -95238,11 +97544,11 @@ pIdxInfo->nOrderBy = nOrderBy; /* Try to find a more efficient access pattern by using multiple indexes ** to optimize an OR expression within the WHERE clause. */ - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } #endif /* SQLITE_OMIT_VIRTUALTABLE */ /* ** Argument pIdx is a pointer to an index structure that has an array of @@ -95364,11 +97670,11 @@ /* The evalConstExpr() function will have already converted any TK_VARIABLE ** expression involved in an comparison into a TK_REGISTER. */ assert( pExpr->op!=TK_VARIABLE ); if( pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE ){ int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); /* IMP: R-23257-02778 */ *pp = sqlite3VdbeGetValue(pParse->pReprepare, iVar, aff); return SQLITE_OK; } return sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, aff, pp); } @@ -95519,11 +97825,12 @@ */ static void bestBtreeIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ Index *pProbe; /* An index we are evaluating */ @@ -95561,27 +97868,18 @@ Index *pFirst; /* Any other index on the table */ memset(&sPk, 0, sizeof(Index)); sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowEst = aiRowEstPk; - aiRowEstPk[1] = 1; sPk.onError = OE_Replace; sPk.pTable = pSrc->pTab; + aiRowEstPk[0] = pSrc->pTab->nRowEst; + aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ sPk.pNext = pFirst; } - /* The aiRowEstPk[0] is an estimate of the total number of rows in the - ** table. Get this information from the ANALYZE information if it is - ** available. If not available, assume the table 1 million rows in size. - */ - if( pFirst ){ - assert( pFirst->aiRowEst!=0 ); /* Allocated together with pFirst */ - aiRowEstPk[0] = pFirst->aiRowEst[0]; - }else{ - aiRowEstPk[0] = 1000000; - } pProbe = &sPk; wsFlagMask = ~( WHERE_COLUMN_IN|WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_RANGE ); eqTermMask = WO_EQ|WO_IN; @@ -95674,11 +97972,11 @@ Expr *pExpr = pTerm->pExpr; wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ nInMul *= 25; bInEst = 1; - }else if( pExpr->x.pList ){ + }else if( ALWAYS(pExpr->x.pList) ){ nInMul *= pExpr->x.pList->nExpr + 1; } }else if( pTerm->eOperator & WO_ISNULL ){ wsFlags |= WHERE_COLUMN_NULL; } @@ -95790,29 +98088,29 @@ ** of output rows, adjust the nRow value accordingly. This only ** matters if the current index is the least costly, so do not bother ** with this step if we already know this index will not be chosen. ** Also, never reduce the output row count below 2 using this step. ** - ** Do not reduce the output row count if pSrc is the only table that - ** is notReady; if notReady is a power of two. This will be the case - ** when the main sqlite3WhereBegin() loop is scanning for a table with - ** and "optimal" index, and on such a scan the output row count - ** reduction is not valid because it does not update the "pCost->used" - ** bitmap. The notReady bitmap will also be a power of two when we - ** are scanning for the last table in a 64-way join. We are willing - ** to bypass this optimization in that corner case. + ** It is critical that the notValid mask be used here instead of + ** the notReady mask. When computing an "optimal" index, the notReady + ** mask will only have one bit set - the bit for the current table. + ** The notValid mask, on the other hand, always has all bits set for + ** tables that are not in outer loops. If notReady is used here instead + ** of notValid, then a optimal index that depends on inner joins loops + ** might be selected even when there exists an optimal index that has + ** no such dependency. */ - if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){ + if( nRow>2 && cost<=pCost->rCost ){ int k; /* Loop counter */ int nSkipEq = nEq; /* Number of == constraints to skip */ int nSkipRange = nBound; /* Number of < constraints to skip */ Bitmask thisTab; /* Bitmap for pSrc */ thisTab = getMask(pWC->pMaskSet, iCur); for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; - if( (pTerm->prereqAll & notReady)!=thisTab ) continue; + if( (pTerm->prereqAll & notValid)!=thisTab ) continue; if( pTerm->eOperator & (WO_EQ|WO_IN|WO_ISNULL) ){ if( nSkipEq ){ /* Ignore the first nEq equality matches since the index ** has already accounted for these */ nSkipEq--; @@ -95850,15 +98148,15 @@ /* If this index is the best we have seen so far, then record this ** index and its cost in the pCost structure. */ if( (!pIdx || wsFlags) - && (costrCost || (cost<=pCost->rCost && nRownRow)) + && (costrCost || (cost<=pCost->rCost && nRowplan.nRow)) ){ pCost->rCost = cost; - pCost->nRow = nRow; pCost->used = used; + pCost->plan.nRow = nRow; pCost->plan.wsFlags = (wsFlags&wsFlagMask); pCost->plan.nEq = nEq; pCost->plan.u.pIdx = pIdx; } @@ -95890,11 +98188,11 @@ WHERETRACE(("best index is: %s\n", ((pCost->plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ? "none" : pCost->plan.u.pIdx ? pCost->plan.u.pIdx->zName : "ipk") )); - bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestOrClauseIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); bestAutomaticIndex(pParse, pWC, pSrc, notReady, pCost); pCost->plan.wsFlags |= eqTermMask; } /* @@ -95905,26 +98203,27 @@ */ static void bestIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ struct SrcList_item *pSrc, /* The FROM clause term to search */ - Bitmask notReady, /* Mask of cursors that are not available */ + Bitmask notReady, /* Mask of cursors not available for indexing */ + Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ WhereCost *pCost /* Lowest cost query plan */ ){ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pSrc->pTab) ){ sqlite3_index_info *p = 0; - bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p); + bestVirtualIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost,&p); if( p->needToFreeIdxStr ){ sqlite3_free(p->idxStr); } sqlite3DbFree(pParse->db, p); }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); } } /* ** Disable a term in the WHERE clause. Except, do not disable the term @@ -95938,10 +98237,13 @@ ** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' ** ** The t2.z='ok' is disabled in the in (2) because it originates ** in the ON clause. The term is disabled in (3) because it is not part ** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** IMPLEMENTATION-OF: R-24597-58655 No tests are done for terms that are +** completely satisfied by indices. ** ** Disabling a term causes that term to not be tested in the inner loop ** of the join. Disabling is an optimization. When terms are satisfied ** by indices, we disable them to prevent redundant tests in the inner ** loop. We would get the correct results if nothing were ever disabled, @@ -96150,10 +98452,11 @@ pTerm = findTerm(pWC, iCur, k, notReady, pLevel->plan.wsFlags, pIdx); if( NEVER(pTerm==0) ) break; /* The following true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ r1 = codeEqualityTerm(pParse, pTerm, pLevel, regBase+j); if( r1!=regBase+j ){ if( nReg==1 ){ sqlite3ReleaseTempReg(pParse, regBase); regBase = r1; @@ -96177,10 +98480,165 @@ } } *pzAff = zAff; return regBase; } + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. +*/ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + int iTerm, /* Index of this term. First is zero */ + const char *zColumn, /* Name of the column */ + const char *zOp /* Name of the operator */ +){ + if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3StrAccumAppend(pStr, zColumn, -1); + sqlite3StrAccumAppend(pStr, zOp, 1); + sqlite3StrAccumAppend(pStr, "?", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function returns a pointer to a string buffer containing a description +** of the subset of table rows scanned by the strategy in the form of an +** SQL expression. Or, if all rows are scanned, NULL is returned. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +** +** The returned pointer points to memory obtained from sqlite3DbMalloc(). +** It is the responsibility of the caller to free the buffer when it is +** no longer required. +*/ +static char *explainIndexRange(sqlite3 *db, WhereLevel *pLevel, Table *pTab){ + WherePlan *pPlan = &pLevel->plan; + Index *pIndex = pPlan->u.pIdx; + int nEq = pPlan->nEq; + int i, j; + Column *aCol = pTab->aCol; + int *aiColumn = pIndex->aiColumn; + StrAccum txt; + + if( nEq==0 && (pPlan->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ + return 0; + } + sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); + txt.db = db; + sqlite3StrAccumAppend(&txt, " (", 2); + for(i=0; i"); + } + if( pPlan->wsFlags&WHERE_TOP_LIMIT ){ + explainAppendTerm(&txt, i, aCol[aiColumn[j]].zName, "<"); + } + sqlite3StrAccumAppend(&txt, ")", 1); + return sqlite3StrAccumFinish(&txt); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single +** record is added to the output to describe the table scan strategy in +** pLevel. +*/ +static void explainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + if( pParse->explain==2 ){ + u32 flags = pLevel->plan.wsFlags; + struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + sqlite3_int64 nRow; /* Expected number of rows visited by scan */ + int iId = pParse->iSelectId; /* Select id (left-most output column) */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + + isSearch = (pLevel->plan.nEq>0) + || (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + if( pItem->pSelect ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + }else{ + zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + } + + if( pItem->zAlias ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + } + if( (flags & WHERE_INDEXED)!=0 ){ + char *zWhere = explainIndexRange(db, pLevel, pItem->pTab); + zMsg = sqlite3MAppendf(db, zMsg, "%s USING %s%sINDEX%s%s%s", zMsg, + ((flags & WHERE_TEMP_INDEX)?"AUTOMATIC ":""), + ((flags & WHERE_IDX_ONLY)?"COVERING ":""), + ((flags & WHERE_TEMP_INDEX)?"":" "), + ((flags & WHERE_TEMP_INDEX)?"": pLevel->plan.u.pIdx->zName), + zWhere + ); + sqlite3DbFree(db, zWhere); + }else if( flags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); + + if( flags&WHERE_ROWID_EQ ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid?)", zMsg); + }else if( flags&WHERE_TOP_LIMIT ){ + zMsg = sqlite3MAppendf(db, zMsg, "%s (rowidplan.u.pVtabIdx; + zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, + pVtabIdx->idxNum, pVtabIdx->idxStr); + } +#endif + if( wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ){ + testcase( wctrlFlags & WHERE_ORDERBY_MIN ); + nRow = 1; + }else{ + nRow = (sqlite3_int64)pLevel->plan.nRow; + } + zMsg = sqlite3MAppendf(db, zMsg, "%s (~%lld rows)", zMsg, nRow); + sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + } +} +#else +# define explainOneScan(u,v,w,x,y,z) +#endif /* SQLITE_OMIT_EXPLAIN */ + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ @@ -96293,10 +98751,11 @@ pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0); assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); assert( pTerm->leftCursor==iCur ); assert( omitTable==0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); @@ -96333,10 +98792,11 @@ }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + testcase( pStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ pX = pStart->pExpr; assert( pX!=0 ); assert( pStart->leftCursor==iCur ); r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); @@ -96350,10 +98810,11 @@ if( pEnd ){ Expr *pX; pX = pEnd->pExpr; assert( pX!=0 ); assert( pEnd->leftCursor==iCur ); + testcase( pEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ memEndValue = ++pParse->nMem; sqlite3ExprCode(pParse, pX->pRight, memEndValue); if( pX->op==TK_LT || pX->op==TK_GT ){ testOp = bRev ? OP_Le : OP_Ge; }else{ @@ -96407,39 +98868,39 @@ ** ** This case is also used when there are no WHERE clause ** constraints but an index is selected anyway, in order ** to force the output order to conform to an ORDER BY. */ - int aStartOp[] = { + static const u8 aStartOp[] = { 0, 0, OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ OP_Last, /* 3: (!start_constraints && startEq && bRev) */ OP_SeekGt, /* 4: (start_constraints && !startEq && !bRev) */ OP_SeekLt, /* 5: (start_constraints && !startEq && bRev) */ OP_SeekGe, /* 6: (start_constraints && startEq && !bRev) */ OP_SeekLe /* 7: (start_constraints && startEq && bRev) */ }; - int aEndOp[] = { + static const u8 aEndOp[] = { OP_Noop, /* 0: (!end_constraints) */ OP_IdxGE, /* 1: (end_constraints && !bRev) */ OP_IdxLT /* 2: (end_constraints && bRev) */ }; - int nEq = pLevel->plan.nEq; + int nEq = pLevel->plan.nEq; /* Number of == or IN terms */ int isMinQuery = 0; /* If this is an optimized SELECT min(x).. */ int regBase; /* Base register holding constraint values */ int r1; /* Temp register */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ int startEq; /* True if range start uses ==, >= or <= */ int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ - int iIdxCur; /* The VDBE cursor for the index */ - int nExtraReg = 0; /* Number of extra registers needed */ - int op; /* Instruction opcode */ + Index *pIdx; /* The index we will be using */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ char *zEndAff; /* Affinity for end of range constraint */ pIdx = pLevel->plan.u.pIdx; iIdxCur = pLevel->iIdxCur; @@ -96517,10 +98978,11 @@ if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ zStartAff[nEq] = SQLITE_AFF_NONE; } } nConstraint++; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ }else if( isMinQuery ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); nConstraint++; startEq = 0; start_constraints = 1; @@ -96556,10 +99018,11 @@ zEndAff[nEq] = SQLITE_AFF_NONE; } } codeApplyAffinity(pParse, regBase, nEq+1, zEndAff); nConstraint++; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); /* EV: R-30575-11662 */ } sqlite3DbFree(pParse->db, zStartAff); sqlite3DbFree(pParse->db, zEndAff); /* Top of the loop body */ @@ -96580,11 +99043,11 @@ ** If it is, jump to the next iteration of the loop. */ r1 = sqlite3GetTempReg(pParse); testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT ); testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT ); - if( pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT) ){ + if( (pLevel->plan.wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){ sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1); sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont); } sqlite3ReleaseTempReg(pParse, r1); @@ -96714,10 +99177,13 @@ /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ + explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); if( (wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); int r; r = sqlite3ExprCodeGetColumn(pParse, pTabItem->pTab, -1, iCur, regRowid); @@ -96762,15 +99228,19 @@ } notReady &= ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. + ** + ** IMPLEMENTATION-OF: R-49525-50935 Terms that cannot be satisfied through + ** the use of indices become tests that are evaluated against each row of + ** the relevant input tables. */ k = 0; for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ Expr *pE; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); @@ -96794,11 +99264,11 @@ pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); VdbeComment((v, "record LEFT JOIN hit")); sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ - testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->prereqAll & notReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; @@ -97012,11 +99482,11 @@ ** subexpression is separated by an AND operator. */ initMaskSet(pMaskSet); whereClauseInit(pWC, pParse, pMaskSet); sqlite3ExprCodeConstants(pParse, pWhere); - whereSplit(pWC, pWhere, TK_AND); + whereSplit(pWC, pWhere, TK_AND); /* IMP: R-15842-53296 */ /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. */ if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ @@ -97100,13 +99570,16 @@ Index *pIdx; /* Index for FROM table at pTabItem */ int j; /* For looping over FROM tables */ int bestJ = -1; /* The value of j */ Bitmask m; /* Bitmask value for j or bestJ */ int isOptimal; /* Iterator for optimal/non-optimal search */ + int nUnconstrained; /* Number tables without INDEXED BY */ + Bitmask notIndexed; /* Mask of tables that cannot use an index */ memset(&bestPlan, 0, sizeof(bestPlan)); bestPlan.rCost = SQLITE_BIG_DBL; + WHERETRACE(("*** Begin search for loop %d ***\n", i)); /* Loop through the remaining entries in the FROM clause to find the ** next nested loop. The loop tests all FROM clause entries ** either once or twice. ** @@ -97120,14 +99593,20 @@ ** by waiting for other tables to run first. This "optimal" test works ** by first assuming that the FROM clause is on the inner loop and finding ** its query plan, then checking to see if that query plan uses any ** other FROM clause terms that are notReady. If no notReady terms are ** used then the "optimal" query plan works. + ** + ** Note that the WhereCost.nRow parameter for an optimal scan might + ** not be as small as it would be if the table really were the innermost + ** join. The nRow value can be reduced by WHERE clause constraints + ** that do not use indices. But this nRow reduction only happens if the + ** table really is the innermost join. ** ** The second loop iteration is only performed if no optimal scan - ** strategies were found by the first loop. This 2nd iteration is used to - ** search for the lowest cost scan overall. + ** strategies were found by the first iteration. This second iteration + ** is used to search for the lowest cost scan overall. ** ** Previous versions of SQLite performed only the second iteration - ** the next outermost loop was always that with the lowest overall ** cost. However, this meant that SQLite could select the wrong plan ** for scripts such as the following: @@ -97136,17 +99615,19 @@ ** CREATE TABLE t2(c, d); ** SELECT * FROM t2, t1 WHERE t2.rowid = t1.a; ** ** The best strategy is to iterate through table t1 first. However it ** is not possible to determine this with a simple greedy algorithm. - ** However, since the cost of a linear scan through table t2 is the same + ** Since the cost of a linear scan through table t2 is the same ** as the cost of a linear scan through table t1, a simple greedy ** algorithm may choose to use t2 for the outer loop, which is a much ** costlier approach. */ - for(isOptimal=(iFrom=0; isOptimal--){ - Bitmask mask; /* Mask of tables not yet ready */ + nUnconstrained = 0; + notIndexed = 0; + for(isOptimal=(iFrom=0 && bestJ<0; isOptimal--){ + Bitmask mask; /* Mask of tables not yet ready */ for(j=iFrom, pTabItem=&pTabList->a[j]; jpIndex==0 ) nUnconstrained++; + WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", + j, isOptimal)); assert( pTabItem->pTab ); #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTabItem->pTab) ){ sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo; - bestVirtualIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost, pp); + bestVirtualIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost, pp); }else #endif { - bestBtreeIndex(pParse, pWC, pTabItem, mask, pOrderBy, &sCost); + bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, + &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); - if( (sCost.used¬Ready)==0 - && (bestJ<0 || sCost.rCostpIndex==0 + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 + || sCost.plan.u.pIdx==pTabItem->pIndex ); + + if( isOptimal && (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)==0 ){ + notIndexed |= m; + } + + /* Conditions under which this table becomes the best so far: + ** + ** (1) The table must not depend on other tables that have not + ** yet run. + ** + ** (2) A full-table-scan plan cannot supercede another plan unless + ** it is an "optimal" plan as defined above. + ** + ** (3) All tables have an INDEXED BY clause or this table lacks an + ** INDEXED BY clause or this table uses the specific + ** index specified by its INDEXED BY clause. This rule ensures + ** that a best-so-far is always selected even if an impossible + ** combination of INDEXED BY clauses are given. The error + ** will be detected and relayed back to the application later. + ** The NEVER() comes about because rule (2) above prevents + ** An indexable full-table-scan from reaching rule (3). + ** + ** (4) The plan cost must be lower than prior plans or else the + ** cost must be the same and the number of rows must be lower. + */ + if( (sCost.used¬Ready)==0 /* (1) */ + && (bestJ<0 || (notIndexed&m)!=0 /* (2) */ + || (sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0) + && (nUnconstrained==0 || pTabItem->pIndex==0 /* (3) */ + || NEVER((sCost.plan.wsFlags & WHERE_NOT_FULLSCAN)!=0)) + && (bestJ<0 || sCost.rCost=0 ); assert( notReady & getMask(pMaskSet, pTabList->a[bestJ].iCursor) ); - WHERETRACE(("*** Optimizer selects table %d for loop %d\n", bestJ, - pLevel-pWInfo->a)); + WHERETRACE(("*** Optimizer selects table %d for loop %d" + " with cost=%g and nRow=%g\n", + bestJ, pLevel-pWInfo->a, bestPlan.rCost, bestPlan.plan.nRow)); if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; @@ -97200,11 +99723,13 @@ }else{ pLevel->iIdxCur = -1; } notReady &= ~getMask(pMaskSet, pTabList->a[bestJ].iCursor); pLevel->iFrom = (u8)bestJ; - if( bestPlan.nRow>=(double)1 ) pParse->nQueryLoop *= bestPlan.nRow; + if( bestPlan.plan.nRow>=(double)1 ){ + pParse->nQueryLoop *= bestPlan.plan.nRow; + } /* Check that if the table scanned by this loop iteration had an ** INDEXED BY clause attached to it, that the named index is being ** used for the scan. If not, then query compilation has failed. ** Return an error. @@ -97248,48 +99773,19 @@ /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ sqlite3CodeVerifySchema(pParse, -1); /* Insert the cookie verifier Goto */ notReady = ~(Bitmask)0; + pWInfo->nRowOut = (double)1; for(i=0, pLevel=pWInfo->a; iexplain==2 ){ - char *zMsg; - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; - zMsg = sqlite3MPrintf(db, "TABLE %s", pItem->zName); - if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); - } - if( (pLevel->plan.wsFlags & WHERE_TEMP_INDEX)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s WITH AUTOMATIC INDEX", zMsg); - }else if( (pLevel->plan.wsFlags & WHERE_INDEXED)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s WITH INDEX %s", - zMsg, pLevel->plan.u.pIdx->zName); - }else if( pLevel->plan.wsFlags & WHERE_MULTI_OR ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s VIA MULTI-INDEX UNION", zMsg); - }else if( pLevel->plan.wsFlags & (WHERE_ROWID_EQ|WHERE_ROWID_RANGE) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING PRIMARY KEY", zMsg); - } -#ifndef SQLITE_OMIT_VIRTUALTABLE - else if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3_index_info *pVtabIdx = pLevel->plan.u.pVtabIdx; - zMsg = sqlite3MAppendf(db, zMsg, "%s VIRTUAL TABLE INDEX %d:%s", zMsg, - pVtabIdx->idxNum, pVtabIdx->idxStr); - } -#endif - if( pLevel->plan.wsFlags & WHERE_ORDERBY ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s ORDER BY", zMsg); - } - sqlite3VdbeAddOp4(v, OP_Explain, i, pLevel->iFrom, 0, zMsg, P4_DYNAMIC); - } -#endif /* SQLITE_OMIT_EXPLAIN */ pTabItem = &pTabList->a[pLevel->iFrom]; pTab = pTabItem->pTab; pLevel->iTabCur = pTabItem->iCursor; + pWInfo->nRowOut *= pLevel->plan.nRow; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ /* Do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -97341,12 +99837,14 @@ ** loop below generates code for a single nested loop of the VM ** program. */ notReady = ~(Bitmask)0; for(i=0; ia[i]; + explainOneScan(pParse, pTabList, pLevel, i, pLevel->iFrom, wctrlFlags); notReady = codeOneLoopStart(pWInfo, i, wctrlFlags, notReady); - pWInfo->iContinue = pWInfo->a[i].addrCont; + pWInfo->iContinue = pLevel->addrCont; } #ifdef SQLITE_TEST /* For testing and debugging use only */ /* Record in the query plan information about the current table ** and the index used to access it (if any). If the table itself @@ -97718,40 +100216,40 @@ ** YYNRULE the number of rules in the grammar ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. */ #define YYCODETYPE unsigned char -#define YYNOCODE 241 +#define YYNOCODE 240 #define YYACTIONTYPE unsigned short int #define YYWILDCARD 67 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - Expr* yy2; - u8 yy18; - ExprSpan yy22; - struct {int value; int mask;} yy47; - SrcList* yy67; - ExprList* yy82; - struct TrigEvent yy210; - IdList* yy240; - struct LimitVal yy244; - TriggerStep* yy347; - int yy412; - struct LikeOp yy438; - Select* yy459; + struct {int value; int mask;} yy13; + u8 yy44; + ExprSpan yy64; + struct TrigEvent yy132; + IdList* yy160; + Expr* yy178; + int yy230; + Select* yy239; + TriggerStep* yy247; + struct LimitVal yy270; + SrcList* yy285; + struct LikeOp yy440; + ExprList* yy462; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define sqlite3ParserARG_SDECL Parse *pParse; #define sqlite3ParserARG_PDECL ,Parse *pParse #define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse #define sqlite3ParserARG_STORE yypParser->pParse = pParse -#define YYNSTATE 561 -#define YYNRULE 290 +#define YYNSTATE 560 +#define YYNRULE 289 #define YYFALLBACK 1 #define YY_NO_ACTION (YYNSTATE+YYNRULE+2) #define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) #define YY_ERROR_ACTION (YYNSTATE+YYNRULE) @@ -97817,438 +100315,435 @@ ** shifting terminals. ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. */ -#define YY_ACTTAB_COUNT (1425) +#define YY_ACTTAB_COUNT (1415) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 281, 42, 486, 39, 135, 151, 558, 528, 48, 48, - /* 10 */ 48, 48, 273, 46, 46, 46, 46, 45, 45, 44, - /* 20 */ 44, 44, 43, 216, 283, 539, 554, 553, 446, 445, - /* 30 */ 521, 515, 41, 46, 46, 46, 46, 45, 45, 44, - /* 40 */ 44, 44, 43, 216, 44, 44, 44, 43, 216, 49, - /* 50 */ 50, 40, 513, 512, 514, 514, 47, 47, 48, 48, - /* 60 */ 48, 48, 195, 46, 46, 46, 46, 45, 45, 44, - /* 70 */ 44, 44, 43, 216, 281, 528, 364, 294, 510, 510, + /* 0 */ 281, 296, 505, 504, 14, 151, 168, 527, 48, 48, + /* 10 */ 48, 48, 41, 46, 46, 46, 46, 45, 45, 44, + /* 20 */ 44, 44, 43, 216, 218, 283, 553, 552, 446, 445, + /* 30 */ 520, 514, 48, 48, 48, 48, 307, 46, 46, 46, + /* 40 */ 46, 45, 45, 44, 44, 44, 43, 216, 557, 49, + /* 50 */ 50, 40, 512, 511, 513, 513, 47, 47, 48, 48, + /* 60 */ 48, 48, 215, 46, 46, 46, 46, 45, 45, 44, + /* 70 */ 44, 44, 43, 216, 281, 527, 364, 294, 509, 509, /* 80 */ 25, 46, 46, 46, 46, 45, 45, 44, 44, 44, - /* 90 */ 43, 216, 294, 602, 551, 550, 145, 366, 319, 337, - /* 100 */ 334, 333, 316, 280, 521, 515, 499, 422, 194, 51, + /* 90 */ 43, 216, 294, 273, 550, 549, 145, 366, 319, 337, + /* 100 */ 334, 333, 316, 280, 520, 514, 498, 422, 194, 51, /* 110 */ 332, 484, 369, 45, 45, 44, 44, 44, 43, 216, - /* 120 */ 483, 532, 83, 49, 50, 40, 513, 512, 514, 514, - /* 130 */ 47, 47, 48, 48, 48, 48, 511, 46, 46, 46, - /* 140 */ 46, 45, 45, 44, 44, 44, 43, 216, 281, 42, - /* 150 */ 204, 39, 135, 121, 156, 127, 255, 340, 250, 339, - /* 160 */ 148, 497, 366, 353, 603, 204, 366, 246, 343, 175, - /* 170 */ 127, 255, 340, 250, 339, 148, 183, 369, 521, 515, - /* 180 */ 145, 369, 246, 337, 334, 333, 532, 82, 533, 456, - /* 190 */ 532, 83, 534, 526, 332, 180, 604, 49, 50, 40, - /* 200 */ 513, 512, 514, 514, 47, 47, 48, 48, 48, 48, - /* 210 */ 531, 46, 46, 46, 46, 45, 45, 44, 44, 44, - /* 220 */ 43, 216, 281, 395, 524, 524, 524, 401, 254, 531, - /* 230 */ 1, 309, 353, 342, 554, 553, 366, 354, 264, 253, - /* 240 */ 145, 179, 132, 337, 334, 333, 552, 554, 553, 373, - /* 250 */ 557, 369, 521, 515, 332, 219, 155, 120, 533, 154, - /* 260 */ 532, 76, 534, 186, 554, 553, 178, 177, 176, 396, - /* 270 */ 37, 49, 50, 40, 513, 512, 514, 514, 47, 47, - /* 280 */ 48, 48, 48, 48, 507, 46, 46, 46, 46, 45, - /* 290 */ 45, 44, 44, 44, 43, 216, 281, 531, 201, 421, - /* 300 */ 351, 211, 551, 550, 59, 366, 399, 498, 554, 553, - /* 310 */ 538, 58, 436, 437, 552, 551, 550, 230, 306, 229, - /* 320 */ 369, 301, 554, 553, 481, 169, 521, 515, 218, 532, - /* 330 */ 83, 495, 551, 550, 172, 412, 430, 226, 146, 31, - /* 340 */ 474, 478, 554, 553, 807, 49, 50, 40, 513, 512, - /* 350 */ 514, 514, 47, 47, 48, 48, 48, 48, 477, 46, + /* 120 */ 483, 531, 82, 49, 50, 40, 512, 511, 513, 513, + /* 130 */ 47, 47, 48, 48, 48, 48, 538, 46, 46, 46, + /* 140 */ 46, 45, 45, 44, 44, 44, 43, 216, 281, 235, + /* 150 */ 204, 436, 437, 121, 156, 127, 255, 340, 250, 339, + /* 160 */ 148, 496, 366, 353, 551, 204, 366, 246, 343, 175, + /* 170 */ 127, 255, 340, 250, 339, 148, 183, 369, 520, 514, + /* 180 */ 145, 369, 246, 337, 334, 333, 531, 75, 527, 456, + /* 190 */ 531, 82, 234, 525, 332, 241, 195, 49, 50, 40, + /* 200 */ 512, 511, 513, 513, 47, 47, 48, 48, 48, 48, + /* 210 */ 530, 46, 46, 46, 46, 45, 45, 44, 44, 44, + /* 220 */ 43, 216, 281, 395, 523, 523, 523, 212, 502, 502, + /* 230 */ 1, 309, 353, 342, 553, 552, 366, 354, 532, 237, + /* 240 */ 145, 239, 533, 337, 334, 333, 551, 553, 552, 373, + /* 250 */ 556, 369, 520, 514, 332, 219, 527, 120, 532, 601, + /* 260 */ 531, 75, 533, 186, 553, 552, 178, 177, 176, 396, + /* 270 */ 602, 49, 50, 40, 512, 511, 513, 513, 47, 47, + /* 280 */ 48, 48, 48, 48, 603, 46, 46, 46, 46, 45, + /* 290 */ 45, 44, 44, 44, 43, 216, 281, 530, 201, 421, + /* 300 */ 351, 211, 550, 549, 58, 366, 193, 132, 553, 552, + /* 310 */ 537, 57, 155, 462, 551, 550, 549, 230, 306, 229, + /* 320 */ 369, 301, 553, 552, 481, 147, 520, 514, 218, 531, + /* 330 */ 82, 454, 550, 549, 172, 412, 151, 226, 527, 31, + /* 340 */ 534, 478, 548, 547, 546, 49, 50, 40, 512, 511, + /* 350 */ 513, 513, 47, 47, 48, 48, 48, 48, 477, 46, /* 360 */ 46, 46, 46, 45, 45, 44, 44, 44, 43, 216, - /* 370 */ 281, 349, 462, 508, 461, 56, 551, 550, 57, 465, - /* 380 */ 196, 223, 528, 225, 463, 276, 275, 274, 158, 272, - /* 390 */ 551, 550, 385, 162, 421, 479, 42, 479, 39, 135, - /* 400 */ 521, 515, 271, 396, 181, 411, 58, 414, 440, 552, - /* 410 */ 551, 550, 53, 28, 415, 4, 554, 553, 170, 49, - /* 420 */ 50, 40, 513, 512, 514, 514, 47, 47, 48, 48, + /* 370 */ 281, 349, 154, 507, 461, 55, 550, 549, 56, 284, + /* 380 */ 196, 223, 527, 225, 463, 276, 275, 274, 158, 272, + /* 390 */ 550, 549, 385, 44, 44, 44, 43, 216, 254, 421, + /* 400 */ 520, 514, 364, 396, 509, 509, 527, 271, 474, 253, + /* 410 */ 411, 57, 53, 28, 551, 510, 553, 552, 170, 49, + /* 420 */ 50, 40, 512, 511, 513, 513, 47, 47, 48, 48, /* 430 */ 48, 48, 396, 46, 46, 46, 46, 45, 45, 44, - /* 440 */ 44, 44, 43, 216, 281, 366, 494, 509, 458, 493, - /* 450 */ 528, 230, 306, 229, 496, 184, 497, 435, 528, 224, - /* 460 */ 369, 364, 218, 510, 510, 134, 472, 134, 150, 532, - /* 470 */ 64, 244, 338, 146, 521, 515, 174, 173, 146, 434, - /* 480 */ 230, 303, 221, 330, 456, 146, 282, 489, 526, 531, - /* 490 */ 479, 218, 479, 49, 50, 40, 513, 512, 514, 514, - /* 500 */ 47, 47, 48, 48, 48, 48, 163, 46, 46, 46, - /* 510 */ 46, 45, 45, 44, 44, 44, 43, 216, 281, 524, - /* 520 */ 524, 524, 441, 136, 403, 492, 528, 329, 404, 292, - /* 530 */ 153, 520, 519, 549, 548, 547, 364, 366, 510, 510, - /* 540 */ 366, 491, 320, 364, 27, 510, 510, 10, 521, 515, - /* 550 */ 11, 218, 369, 517, 516, 369, 443, 490, 147, 357, - /* 560 */ 188, 532, 76, 307, 532, 68, 119, 49, 50, 40, - /* 570 */ 513, 512, 514, 514, 47, 47, 48, 48, 48, 48, - /* 580 */ 518, 46, 46, 46, 46, 45, 45, 44, 44, 44, - /* 590 */ 43, 216, 281, 492, 531, 364, 243, 510, 510, 561, - /* 600 */ 559, 298, 212, 182, 366, 296, 506, 505, 345, 491, - /* 610 */ 286, 366, 284, 362, 506, 505, 346, 246, 308, 369, - /* 620 */ 188, 476, 521, 515, 23, 490, 369, 450, 532, 68, - /* 630 */ 852, 165, 374, 2, 55, 532, 63, 531, 449, 305, - /* 640 */ 241, 49, 50, 40, 513, 512, 514, 514, 47, 47, - /* 650 */ 48, 48, 48, 48, 531, 46, 46, 46, 46, 45, - /* 660 */ 45, 44, 44, 44, 43, 216, 281, 215, 366, 149, - /* 670 */ 366, 405, 405, 322, 285, 215, 366, 151, 308, 528, - /* 680 */ 499, 361, 194, 369, 237, 369, 239, 366, 470, 366, - /* 690 */ 314, 369, 532, 60, 532, 85, 521, 515, 130, 315, - /* 700 */ 532, 88, 369, 552, 369, 552, 400, 389, 146, 150, - /* 710 */ 131, 532, 86, 532, 94, 49, 50, 40, 513, 512, - /* 720 */ 514, 514, 47, 47, 48, 48, 48, 48, 366, 46, + /* 440 */ 44, 44, 43, 216, 281, 366, 247, 465, 458, 345, + /* 450 */ 527, 230, 306, 229, 508, 184, 496, 346, 37, 224, + /* 460 */ 369, 551, 218, 435, 42, 39, 135, 495, 479, 531, + /* 470 */ 63, 405, 405, 322, 520, 514, 174, 173, 338, 146, + /* 480 */ 230, 303, 221, 291, 456, 434, 282, 401, 525, 530, + /* 490 */ 472, 218, 150, 49, 50, 40, 512, 511, 513, 513, + /* 500 */ 47, 47, 48, 48, 48, 48, 506, 46, 46, 46, + /* 510 */ 46, 45, 45, 44, 44, 44, 43, 216, 281, 523, + /* 520 */ 523, 523, 441, 163, 403, 312, 366, 162, 404, 479, + /* 530 */ 497, 519, 518, 313, 430, 292, 146, 366, 134, 136, + /* 540 */ 366, 369, 320, 364, 169, 509, 509, 181, 520, 514, + /* 550 */ 531, 81, 369, 516, 515, 369, 364, 443, 509, 509, + /* 560 */ 182, 531, 67, 479, 531, 67, 399, 49, 50, 40, + /* 570 */ 512, 511, 513, 513, 47, 47, 48, 48, 48, 48, + /* 580 */ 517, 46, 46, 46, 46, 45, 45, 44, 44, 44, + /* 590 */ 43, 216, 281, 491, 530, 494, 243, 491, 244, 134, + /* 600 */ 146, 264, 553, 552, 805, 329, 330, 286, 146, 490, + /* 610 */ 285, 314, 364, 490, 509, 509, 362, 505, 504, 560, + /* 620 */ 558, 298, 520, 514, 479, 489, 551, 450, 400, 489, + /* 630 */ 146, 357, 493, 188, 180, 188, 147, 32, 449, 30, + /* 640 */ 23, 49, 50, 40, 512, 511, 513, 513, 47, 47, + /* 650 */ 48, 48, 48, 48, 192, 46, 46, 46, 46, 45, + /* 660 */ 45, 44, 44, 44, 43, 216, 281, 530, 530, 530, + /* 670 */ 550, 549, 850, 165, 374, 2, 389, 215, 150, 299, + /* 680 */ 179, 132, 414, 440, 366, 42, 39, 135, 530, 415, + /* 690 */ 284, 308, 470, 308, 153, 246, 520, 514, 218, 369, + /* 700 */ 202, 492, 498, 361, 194, 217, 54, 551, 531, 62, + /* 710 */ 300, 305, 15, 315, 27, 49, 50, 40, 512, 511, + /* 720 */ 513, 513, 47, 47, 48, 48, 48, 48, 366, 46, /* 730 */ 46, 46, 46, 45, 45, 44, 44, 44, 43, 216, - /* 740 */ 281, 366, 247, 369, 366, 299, 388, 528, 16, 503, - /* 750 */ 503, 295, 532, 92, 537, 423, 369, 552, 366, 369, - /* 760 */ 366, 468, 366, 426, 218, 532, 96, 21, 532, 97, - /* 770 */ 521, 515, 377, 369, 16, 369, 300, 369, 209, 291, - /* 780 */ 202, 433, 532, 122, 532, 123, 532, 54, 147, 49, - /* 790 */ 38, 40, 513, 512, 514, 514, 47, 47, 48, 48, + /* 740 */ 281, 366, 388, 369, 16, 247, 191, 187, 366, 536, + /* 750 */ 423, 4, 531, 59, 366, 295, 369, 488, 426, 366, + /* 760 */ 551, 366, 470, 369, 11, 531, 84, 559, 2, 369, + /* 770 */ 520, 514, 531, 87, 369, 10, 369, 551, 531, 85, + /* 780 */ 530, 530, 203, 531, 94, 531, 93, 452, 451, 49, + /* 790 */ 38, 40, 512, 511, 513, 513, 47, 47, 48, 48, /* 800 */ 48, 48, 366, 46, 46, 46, 46, 45, 45, 44, - /* 810 */ 44, 44, 43, 216, 281, 366, 247, 369, 217, 14, - /* 820 */ 247, 168, 314, 366, 312, 15, 532, 93, 309, 192, - /* 830 */ 369, 552, 313, 536, 366, 552, 366, 552, 369, 532, - /* 840 */ 91, 426, 284, 552, 521, 515, 460, 532, 84, 369, - /* 850 */ 32, 369, 30, 203, 193, 132, 191, 290, 532, 90, - /* 860 */ 532, 67, 187, 531, 50, 40, 513, 512, 514, 514, + /* 810 */ 44, 44, 43, 216, 281, 366, 247, 369, 366, 247, + /* 820 */ 247, 474, 314, 309, 185, 321, 531, 96, 398, 451, + /* 830 */ 369, 551, 470, 369, 551, 551, 119, 551, 551, 531, + /* 840 */ 97, 149, 531, 122, 520, 514, 554, 551, 314, 377, + /* 850 */ 476, 16, 468, 290, 474, 474, 289, 288, 530, 460, + /* 860 */ 210, 551, 459, 551, 50, 40, 512, 511, 513, 513, /* 870 */ 47, 47, 48, 48, 48, 48, 366, 46, 46, 46, /* 880 */ 46, 45, 45, 44, 44, 44, 43, 216, 281, 366, - /* 890 */ 531, 369, 366, 144, 247, 197, 531, 560, 2, 474, - /* 900 */ 532, 87, 452, 451, 369, 366, 555, 369, 366, 552, - /* 910 */ 459, 366, 228, 532, 126, 117, 532, 125, 521, 515, - /* 920 */ 369, 552, 314, 369, 398, 451, 369, 336, 474, 532, - /* 930 */ 124, 289, 532, 66, 474, 532, 81, 552, 210, 40, - /* 940 */ 513, 512, 514, 514, 47, 47, 48, 48, 48, 48, - /* 950 */ 185, 46, 46, 46, 46, 45, 45, 44, 44, 44, - /* 960 */ 43, 216, 36, 360, 470, 3, 366, 259, 245, 370, - /* 970 */ 553, 116, 366, 257, 43, 216, 501, 214, 20, 552, - /* 980 */ 363, 369, 331, 366, 531, 36, 360, 369, 3, 366, - /* 990 */ 532, 80, 370, 553, 559, 298, 532, 65, 369, 358, - /* 1000 */ 366, 543, 247, 363, 369, 115, 235, 532, 79, 497, - /* 1010 */ 253, 542, 200, 532, 78, 369, 552, 552, 366, 235, - /* 1020 */ 366, 552, 358, 327, 532, 89, 552, 321, 409, 33, - /* 1030 */ 34, 464, 497, 369, 552, 369, 35, 368, 367, 288, - /* 1040 */ 235, 526, 532, 77, 532, 75, 167, 406, 541, 234, - /* 1050 */ 235, 113, 33, 34, 366, 552, 366, 323, 454, 35, - /* 1060 */ 368, 367, 384, 552, 526, 552, 112, 535, 150, 369, - /* 1070 */ 402, 369, 524, 524, 524, 523, 522, 12, 532, 74, - /* 1080 */ 532, 73, 218, 382, 36, 360, 128, 3, 366, 365, - /* 1090 */ 232, 370, 553, 381, 366, 524, 524, 524, 523, 522, - /* 1100 */ 12, 552, 363, 369, 552, 366, 139, 26, 360, 369, - /* 1110 */ 3, 366, 532, 62, 370, 553, 111, 13, 532, 61, - /* 1120 */ 369, 358, 525, 394, 138, 363, 369, 391, 470, 532, - /* 1130 */ 72, 497, 366, 355, 235, 532, 71, 552, 488, 137, - /* 1140 */ 366, 109, 108, 552, 358, 18, 344, 369, 552, 552, - /* 1150 */ 17, 33, 34, 552, 497, 369, 532, 70, 35, 368, - /* 1160 */ 367, 369, 9, 526, 532, 69, 106, 386, 383, 235, - /* 1170 */ 532, 7, 101, 235, 33, 34, 99, 380, 475, 98, - /* 1180 */ 8, 35, 368, 367, 552, 235, 526, 471, 552, 469, - /* 1190 */ 249, 287, 439, 552, 524, 524, 524, 523, 522, 12, - /* 1200 */ 552, 376, 552, 297, 552, 552, 375, 552, 248, 419, - /* 1210 */ 242, 240, 199, 95, 238, 160, 198, 524, 524, 524, - /* 1220 */ 523, 522, 12, 552, 552, 552, 552, 236, 220, 552, - /* 1230 */ 231, 317, 393, 392, 310, 263, 208, 133, 348, 544, - /* 1240 */ 262, 6, 552, 556, 546, 552, 552, 552, 552, 552, - /* 1250 */ 552, 159, 552, 552, 114, 552, 545, 372, 371, 530, - /* 1260 */ 157, 218, 23, 266, 350, 352, 24, 268, 29, 267, - /* 1270 */ 213, 52, 265, 487, 152, 164, 529, 22, 347, 453, - /* 1280 */ 341, 448, 293, 443, 444, 260, 118, 473, 258, 251, - /* 1290 */ 467, 442, 143, 279, 328, 447, 206, 418, 417, 256, - /* 1300 */ 416, 424, 466, 142, 278, 410, 326, 141, 324, 190, - /* 1310 */ 408, 129, 19, 140, 107, 318, 397, 110, 387, 304, - /* 1320 */ 540, 105, 104, 207, 103, 102, 205, 527, 216, 853, - /* 1330 */ 277, 166, 359, 269, 233, 428, 379, 485, 171, 407, - /* 1340 */ 482, 853, 161, 252, 440, 429, 425, 427, 100, 853, - /* 1350 */ 335, 500, 853, 853, 853, 853, 853, 853, 504, 261, - /* 1360 */ 853, 853, 853, 457, 853, 853, 853, 853, 455, 853, - /* 1370 */ 853, 853, 390, 438, 853, 432, 431, 502, 853, 5, - /* 1380 */ 378, 270, 356, 420, 853, 413, 222, 853, 302, 853, - /* 1390 */ 853, 853, 480, 853, 853, 853, 853, 853, 853, 853, - /* 1400 */ 853, 853, 853, 853, 853, 853, 311, 853, 853, 853, - /* 1410 */ 853, 853, 853, 853, 853, 853, 853, 227, 853, 853, - /* 1420 */ 853, 853, 189, 853, 325, + /* 890 */ 197, 369, 366, 259, 257, 287, 131, 366, 535, 130, + /* 900 */ 531, 123, 43, 216, 369, 366, 426, 369, 366, 542, + /* 910 */ 366, 228, 369, 531, 88, 209, 531, 92, 520, 514, + /* 920 */ 369, 531, 91, 369, 551, 369, 500, 214, 433, 531, + /* 930 */ 83, 21, 531, 90, 531, 66, 336, 200, 117, 40, + /* 940 */ 512, 511, 513, 513, 47, 47, 48, 48, 48, 48, + /* 950 */ 245, 46, 46, 46, 46, 45, 45, 44, 44, 44, + /* 960 */ 43, 216, 36, 360, 541, 3, 366, 116, 235, 370, + /* 970 */ 552, 20, 235, 331, 558, 298, 115, 144, 327, 551, + /* 980 */ 363, 369, 366, 551, 409, 36, 360, 551, 3, 366, + /* 990 */ 531, 86, 370, 552, 253, 540, 167, 369, 113, 358, + /* 1000 */ 112, 323, 128, 363, 369, 402, 531, 126, 150, 496, + /* 1010 */ 551, 384, 235, 531, 125, 220, 232, 551, 366, 235, + /* 1020 */ 366, 406, 358, 139, 366, 235, 111, 551, 13, 33, + /* 1030 */ 34, 464, 496, 369, 551, 369, 35, 368, 367, 369, + /* 1040 */ 551, 525, 531, 124, 531, 65, 394, 138, 531, 80, + /* 1050 */ 235, 137, 33, 34, 366, 382, 366, 391, 109, 35, + /* 1060 */ 368, 367, 381, 18, 525, 551, 17, 106, 380, 369, + /* 1070 */ 9, 369, 523, 523, 523, 522, 521, 12, 531, 79, + /* 1080 */ 531, 64, 218, 108, 36, 360, 365, 3, 366, 235, + /* 1090 */ 383, 370, 552, 199, 366, 523, 523, 523, 522, 521, + /* 1100 */ 12, 551, 363, 369, 551, 366, 386, 26, 360, 369, + /* 1110 */ 3, 366, 531, 78, 370, 552, 101, 99, 531, 77, + /* 1120 */ 369, 358, 98, 8, 376, 363, 369, 297, 524, 531, + /* 1130 */ 89, 496, 198, 375, 355, 531, 76, 95, 208, 114, + /* 1140 */ 366, 487, 366, 551, 358, 160, 366, 555, 545, 551, + /* 1150 */ 544, 33, 34, 366, 496, 369, 551, 369, 35, 368, + /* 1160 */ 367, 369, 372, 525, 531, 74, 531, 73, 369, 475, + /* 1170 */ 531, 72, 543, 471, 33, 34, 469, 531, 61, 371, + /* 1180 */ 159, 35, 368, 367, 551, 366, 525, 366, 551, 249, + /* 1190 */ 366, 551, 439, 6, 523, 523, 523, 522, 521, 12, + /* 1200 */ 369, 366, 369, 157, 551, 369, 529, 551, 366, 531, + /* 1210 */ 60, 531, 71, 218, 531, 70, 369, 523, 523, 523, + /* 1220 */ 522, 521, 12, 369, 344, 531, 69, 248, 419, 242, + /* 1230 */ 240, 238, 531, 68, 23, 236, 267, 231, 317, 369, + /* 1240 */ 393, 392, 551, 551, 551, 551, 551, 310, 531, 7, + /* 1250 */ 551, 263, 551, 551, 133, 551, 551, 348, 262, 268, + /* 1260 */ 350, 265, 551, 266, 24, 352, 551, 528, 29, 551, + /* 1270 */ 52, 213, 551, 551, 486, 152, 164, 260, 22, 347, + /* 1280 */ 473, 258, 341, 467, 466, 256, 453, 448, 293, 447, + /* 1290 */ 251, 444, 443, 118, 442, 279, 206, 418, 143, 207, + /* 1300 */ 417, 416, 278, 328, 424, 205, 410, 142, 324, 326, + /* 1310 */ 141, 19, 129, 166, 107, 190, 408, 140, 233, 318, + /* 1320 */ 407, 397, 110, 304, 539, 387, 105, 379, 104, 526, + /* 1330 */ 485, 103, 102, 100, 161, 428, 311, 482, 227, 261, + /* 1340 */ 503, 171, 252, 501, 269, 429, 440, 427, 425, 390, + /* 1350 */ 499, 335, 277, 216, 851, 480, 359, 222, 851, 851, + /* 1360 */ 851, 851, 457, 455, 851, 851, 438, 851, 432, 431, + /* 1370 */ 851, 851, 356, 5, 378, 270, 302, 420, 413, 851, + /* 1380 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, + /* 1390 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, + /* 1400 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, + /* 1410 */ 851, 851, 189, 851, 325, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 220, 221, 222, 223, 24, 1, 26, 77, 78, - /* 10 */ 79, 80, 15, 82, 83, 84, 85, 86, 87, 88, - /* 20 */ 89, 90, 91, 92, 153, 23, 26, 27, 7, 8, - /* 30 */ 49, 50, 81, 82, 83, 84, 85, 86, 87, 88, - /* 40 */ 89, 90, 91, 92, 88, 89, 90, 91, 92, 68, + /* 0 */ 19, 167, 168, 169, 22, 24, 24, 26, 77, 78, + /* 10 */ 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + /* 20 */ 89, 90, 91, 92, 116, 153, 26, 27, 7, 8, + /* 30 */ 49, 50, 77, 78, 79, 80, 128, 82, 83, 84, + /* 40 */ 85, 86, 87, 88, 89, 90, 91, 92, 1, 68, /* 50 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - /* 60 */ 79, 80, 22, 82, 83, 84, 85, 86, 87, 88, + /* 60 */ 79, 80, 228, 82, 83, 84, 85, 86, 87, 88, /* 70 */ 89, 90, 91, 92, 19, 94, 112, 19, 114, 115, /* 80 */ 25, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 90 */ 91, 92, 19, 118, 94, 95, 96, 148, 227, 99, - /* 100 */ 100, 101, 231, 161, 49, 50, 164, 165, 166, 54, + /* 90 */ 91, 92, 19, 15, 94, 95, 96, 148, 226, 99, + /* 100 */ 100, 101, 230, 161, 49, 50, 164, 165, 166, 54, /* 110 */ 110, 32, 163, 86, 87, 88, 89, 90, 91, 92, /* 120 */ 41, 172, 173, 68, 69, 70, 71, 72, 73, 74, - /* 130 */ 75, 76, 77, 78, 79, 80, 113, 82, 83, 84, - /* 140 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 220, - /* 150 */ 92, 222, 223, 24, 96, 97, 98, 99, 100, 101, - /* 160 */ 102, 66, 148, 214, 118, 92, 148, 109, 219, 25, + /* 130 */ 75, 76, 77, 78, 79, 80, 23, 82, 83, 84, + /* 140 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 148, + /* 150 */ 92, 97, 98, 24, 96, 97, 98, 99, 100, 101, + /* 160 */ 102, 66, 148, 214, 163, 92, 148, 109, 219, 25, /* 170 */ 97, 98, 99, 100, 101, 102, 158, 163, 49, 50, - /* 180 */ 96, 163, 109, 99, 100, 101, 172, 173, 113, 94, - /* 190 */ 172, 173, 117, 98, 110, 158, 118, 68, 69, 70, + /* 180 */ 96, 163, 109, 99, 100, 101, 172, 173, 26, 94, + /* 190 */ 172, 173, 191, 98, 110, 16, 22, 68, 69, 70, /* 200 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, /* 210 */ 192, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 220 */ 91, 92, 19, 11, 129, 130, 131, 21, 98, 192, - /* 230 */ 22, 148, 214, 88, 26, 27, 148, 219, 224, 109, - /* 240 */ 96, 204, 205, 99, 100, 101, 163, 26, 27, 144, - /* 250 */ 145, 163, 49, 50, 110, 150, 118, 152, 113, 118, + /* 220 */ 91, 92, 19, 11, 129, 130, 131, 213, 129, 130, + /* 230 */ 22, 148, 214, 88, 26, 27, 148, 219, 113, 60, + /* 240 */ 96, 62, 117, 99, 100, 101, 163, 26, 27, 144, + /* 250 */ 145, 163, 49, 50, 110, 150, 94, 152, 113, 118, /* 260 */ 172, 173, 117, 158, 26, 27, 105, 106, 107, 57, - /* 270 */ 22, 68, 69, 70, 71, 72, 73, 74, 75, 76, - /* 280 */ 77, 78, 79, 80, 23, 82, 83, 84, 85, 86, + /* 270 */ 118, 68, 69, 70, 71, 72, 73, 74, 75, 76, + /* 280 */ 77, 78, 79, 80, 118, 82, 83, 84, 85, 86, /* 290 */ 87, 88, 89, 90, 91, 92, 19, 192, 215, 148, - /* 300 */ 212, 213, 94, 95, 22, 148, 100, 23, 26, 27, - /* 310 */ 159, 160, 97, 98, 163, 94, 95, 105, 106, 107, - /* 320 */ 163, 216, 26, 27, 25, 22, 49, 50, 116, 172, - /* 330 */ 173, 11, 94, 95, 183, 184, 23, 16, 25, 136, - /* 340 */ 164, 120, 26, 27, 138, 68, 69, 70, 71, 72, + /* 300 */ 212, 213, 94, 95, 22, 148, 204, 205, 26, 27, + /* 310 */ 159, 160, 118, 23, 163, 94, 95, 105, 106, 107, + /* 320 */ 163, 216, 26, 27, 25, 50, 49, 50, 116, 172, + /* 330 */ 173, 163, 94, 95, 183, 184, 24, 16, 26, 136, + /* 340 */ 172, 120, 7, 8, 9, 68, 69, 70, 71, 72, /* 350 */ 73, 74, 75, 76, 77, 78, 79, 80, 120, 82, /* 360 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 370 */ 19, 214, 23, 23, 23, 25, 94, 95, 22, 203, + /* 370 */ 19, 214, 118, 23, 23, 25, 94, 95, 22, 104, /* 380 */ 5, 60, 26, 62, 88, 10, 11, 12, 13, 14, - /* 390 */ 94, 95, 17, 23, 148, 25, 220, 25, 222, 223, - /* 400 */ 49, 50, 156, 57, 22, 159, 160, 179, 180, 163, - /* 410 */ 94, 95, 22, 136, 186, 35, 26, 27, 119, 68, + /* 390 */ 94, 95, 17, 88, 89, 90, 91, 92, 98, 148, + /* 400 */ 49, 50, 112, 57, 114, 115, 94, 156, 164, 109, + /* 410 */ 159, 160, 22, 136, 163, 113, 26, 27, 119, 68, /* 420 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, /* 430 */ 79, 80, 57, 82, 83, 84, 85, 86, 87, 88, - /* 440 */ 89, 90, 91, 92, 19, 148, 23, 23, 23, 23, - /* 450 */ 94, 105, 106, 107, 23, 158, 66, 36, 26, 138, - /* 460 */ 163, 112, 116, 114, 115, 95, 23, 95, 25, 172, - /* 470 */ 173, 23, 51, 25, 49, 50, 86, 87, 25, 58, - /* 480 */ 105, 106, 107, 23, 94, 25, 111, 23, 98, 192, - /* 490 */ 120, 116, 120, 68, 69, 70, 71, 72, 73, 74, + /* 440 */ 89, 90, 91, 92, 19, 148, 148, 203, 23, 19, + /* 450 */ 94, 105, 106, 107, 23, 158, 66, 27, 22, 138, + /* 460 */ 163, 163, 116, 36, 220, 221, 222, 23, 25, 172, + /* 470 */ 173, 105, 106, 107, 49, 50, 86, 87, 51, 25, + /* 480 */ 105, 106, 107, 185, 94, 58, 111, 21, 98, 192, + /* 490 */ 23, 116, 25, 68, 69, 70, 71, 72, 73, 74, /* 500 */ 75, 76, 77, 78, 79, 80, 23, 82, 83, 84, /* 510 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 129, - /* 520 */ 130, 131, 23, 25, 30, 12, 94, 19, 34, 108, - /* 530 */ 25, 49, 50, 7, 8, 9, 112, 148, 114, 115, - /* 540 */ 148, 28, 48, 112, 25, 114, 115, 35, 49, 50, - /* 550 */ 25, 116, 163, 71, 72, 163, 103, 44, 50, 46, - /* 560 */ 158, 172, 173, 128, 172, 173, 22, 68, 69, 70, + /* 520 */ 130, 131, 23, 23, 30, 19, 148, 23, 34, 25, + /* 530 */ 23, 49, 50, 27, 23, 108, 25, 148, 95, 25, + /* 540 */ 148, 163, 48, 112, 22, 114, 115, 22, 49, 50, + /* 550 */ 172, 173, 163, 71, 72, 163, 112, 103, 114, 115, + /* 560 */ 158, 172, 173, 120, 172, 173, 100, 68, 69, 70, /* 570 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, /* 580 */ 98, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 590 */ 91, 92, 19, 12, 192, 112, 23, 114, 115, 0, - /* 600 */ 1, 2, 213, 158, 148, 167, 168, 169, 19, 28, - /* 610 */ 218, 148, 104, 167, 168, 169, 27, 109, 216, 163, - /* 620 */ 158, 120, 49, 50, 126, 44, 163, 46, 172, 173, - /* 630 */ 140, 141, 142, 143, 232, 172, 173, 192, 57, 237, - /* 640 */ 16, 68, 69, 70, 71, 72, 73, 74, 75, 76, - /* 650 */ 77, 78, 79, 80, 192, 82, 83, 84, 85, 86, - /* 660 */ 87, 88, 89, 90, 91, 92, 19, 229, 148, 35, - /* 670 */ 148, 105, 106, 107, 218, 229, 148, 24, 216, 26, - /* 680 */ 164, 165, 166, 163, 60, 163, 62, 148, 148, 148, - /* 690 */ 148, 163, 172, 173, 172, 173, 49, 50, 39, 237, - /* 700 */ 172, 173, 163, 163, 163, 163, 23, 23, 25, 25, - /* 710 */ 118, 172, 173, 172, 173, 68, 69, 70, 71, 72, + /* 590 */ 91, 92, 19, 12, 192, 11, 23, 12, 23, 95, + /* 600 */ 25, 223, 26, 27, 138, 19, 23, 218, 25, 28, + /* 610 */ 218, 148, 112, 28, 114, 115, 167, 168, 169, 0, + /* 620 */ 1, 2, 49, 50, 120, 44, 163, 46, 23, 44, + /* 630 */ 25, 46, 23, 158, 158, 158, 50, 135, 57, 137, + /* 640 */ 126, 68, 69, 70, 71, 72, 73, 74, 75, 76, + /* 650 */ 77, 78, 79, 80, 158, 82, 83, 84, 85, 86, + /* 660 */ 87, 88, 89, 90, 91, 92, 19, 192, 192, 192, + /* 670 */ 94, 95, 140, 141, 142, 143, 23, 228, 25, 97, + /* 680 */ 204, 205, 179, 180, 148, 220, 221, 222, 192, 186, + /* 690 */ 104, 216, 148, 216, 25, 109, 49, 50, 116, 163, + /* 700 */ 237, 23, 164, 165, 166, 195, 231, 163, 172, 173, + /* 710 */ 128, 236, 202, 236, 25, 68, 69, 70, 71, 72, /* 720 */ 73, 74, 75, 76, 77, 78, 79, 80, 148, 82, /* 730 */ 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, - /* 740 */ 19, 148, 148, 163, 148, 97, 23, 94, 25, 129, - /* 750 */ 130, 211, 172, 173, 170, 171, 163, 163, 148, 163, - /* 760 */ 148, 27, 148, 179, 116, 172, 173, 22, 172, 173, - /* 770 */ 49, 50, 23, 163, 25, 163, 128, 163, 52, 185, - /* 780 */ 238, 29, 172, 173, 172, 173, 172, 173, 50, 68, + /* 740 */ 19, 148, 23, 163, 25, 148, 158, 158, 148, 170, + /* 750 */ 171, 35, 172, 173, 148, 211, 163, 23, 179, 148, + /* 760 */ 163, 148, 148, 163, 25, 172, 173, 142, 143, 163, + /* 770 */ 49, 50, 172, 173, 163, 35, 163, 163, 172, 173, + /* 780 */ 192, 192, 185, 172, 173, 172, 173, 188, 189, 68, /* 790 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, /* 800 */ 79, 80, 148, 82, 83, 84, 85, 86, 87, 88, - /* 810 */ 89, 90, 91, 92, 19, 148, 148, 163, 195, 22, - /* 820 */ 148, 24, 148, 148, 19, 202, 172, 173, 148, 158, - /* 830 */ 163, 163, 27, 171, 148, 163, 148, 163, 163, 172, - /* 840 */ 173, 179, 104, 163, 49, 50, 23, 172, 173, 163, - /* 850 */ 135, 163, 137, 185, 204, 205, 158, 185, 172, 173, - /* 860 */ 172, 173, 158, 192, 69, 70, 71, 72, 73, 74, + /* 810 */ 89, 90, 91, 92, 19, 148, 148, 163, 148, 148, + /* 820 */ 148, 164, 148, 148, 158, 211, 172, 173, 188, 189, + /* 830 */ 163, 163, 148, 163, 163, 163, 22, 163, 163, 172, + /* 840 */ 173, 35, 172, 173, 49, 50, 148, 163, 148, 23, + /* 850 */ 120, 25, 27, 185, 164, 164, 185, 185, 192, 23, + /* 860 */ 203, 163, 23, 163, 69, 70, 71, 72, 73, 74, /* 870 */ 75, 76, 77, 78, 79, 80, 148, 82, 83, 84, /* 880 */ 85, 86, 87, 88, 89, 90, 91, 92, 19, 148, - /* 890 */ 192, 163, 148, 102, 148, 215, 192, 142, 143, 164, - /* 900 */ 172, 173, 188, 189, 163, 148, 148, 163, 148, 163, - /* 910 */ 23, 148, 238, 172, 173, 22, 172, 173, 49, 50, - /* 920 */ 163, 163, 148, 163, 188, 189, 163, 52, 164, 172, - /* 930 */ 173, 185, 172, 173, 164, 172, 173, 163, 203, 70, + /* 890 */ 215, 163, 148, 203, 203, 211, 118, 148, 171, 39, + /* 900 */ 172, 173, 91, 92, 163, 148, 179, 163, 148, 148, + /* 910 */ 148, 237, 163, 172, 173, 52, 172, 173, 49, 50, + /* 920 */ 163, 172, 173, 163, 163, 163, 86, 87, 29, 172, + /* 930 */ 173, 22, 172, 173, 172, 173, 52, 237, 22, 70, /* 940 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, - /* 950 */ 158, 82, 83, 84, 85, 86, 87, 88, 89, 90, - /* 960 */ 91, 92, 19, 20, 148, 22, 148, 203, 23, 26, - /* 970 */ 27, 22, 148, 203, 91, 92, 86, 87, 22, 163, - /* 980 */ 37, 163, 52, 148, 192, 19, 20, 163, 22, 148, - /* 990 */ 172, 173, 26, 27, 1, 2, 172, 173, 163, 56, - /* 1000 */ 148, 148, 148, 37, 163, 22, 148, 172, 173, 66, - /* 1010 */ 109, 148, 238, 172, 173, 163, 163, 163, 148, 148, - /* 1020 */ 148, 163, 56, 19, 172, 173, 163, 211, 20, 86, - /* 1030 */ 87, 88, 66, 163, 163, 163, 93, 94, 95, 185, - /* 1040 */ 148, 98, 172, 173, 172, 173, 24, 59, 148, 191, - /* 1050 */ 148, 104, 86, 87, 148, 163, 148, 43, 163, 93, - /* 1060 */ 94, 95, 191, 163, 98, 163, 53, 172, 25, 163, - /* 1070 */ 53, 163, 129, 130, 131, 132, 133, 134, 172, 173, - /* 1080 */ 172, 173, 116, 191, 19, 20, 148, 22, 148, 148, - /* 1090 */ 138, 26, 27, 191, 148, 129, 130, 131, 132, 133, - /* 1100 */ 134, 163, 37, 163, 163, 148, 104, 19, 20, 163, - /* 1110 */ 22, 148, 172, 173, 26, 27, 22, 5, 172, 173, - /* 1120 */ 163, 56, 148, 1, 118, 37, 163, 27, 148, 172, - /* 1130 */ 173, 66, 148, 148, 148, 172, 173, 163, 148, 35, - /* 1140 */ 148, 108, 127, 163, 56, 76, 148, 163, 163, 163, - /* 1150 */ 76, 86, 87, 163, 66, 163, 172, 173, 93, 94, - /* 1160 */ 95, 163, 22, 98, 172, 173, 119, 1, 20, 148, - /* 1170 */ 172, 173, 119, 148, 86, 87, 108, 191, 148, 127, - /* 1180 */ 22, 93, 94, 95, 163, 148, 98, 148, 163, 148, - /* 1190 */ 148, 211, 148, 163, 129, 130, 131, 132, 133, 134, - /* 1200 */ 163, 128, 163, 3, 163, 163, 4, 163, 148, 148, - /* 1210 */ 148, 148, 191, 162, 148, 6, 191, 129, 130, 131, - /* 1220 */ 132, 133, 134, 163, 163, 163, 163, 148, 191, 163, - /* 1230 */ 148, 148, 148, 148, 148, 148, 178, 148, 148, 13, - /* 1240 */ 148, 25, 163, 147, 147, 163, 163, 163, 163, 163, - /* 1250 */ 163, 149, 163, 163, 178, 163, 147, 147, 157, 192, - /* 1260 */ 149, 116, 126, 199, 123, 122, 124, 197, 135, 198, - /* 1270 */ 225, 125, 200, 155, 118, 155, 201, 104, 121, 174, - /* 1280 */ 104, 174, 47, 103, 176, 208, 22, 209, 208, 174, - /* 1290 */ 209, 174, 154, 177, 18, 182, 92, 174, 174, 208, - /* 1300 */ 174, 182, 209, 154, 177, 155, 155, 154, 45, 155, - /* 1310 */ 155, 68, 135, 154, 22, 155, 187, 187, 197, 18, - /* 1320 */ 151, 190, 190, 228, 190, 190, 228, 164, 92, 240, - /* 1330 */ 146, 217, 226, 196, 235, 164, 197, 175, 194, 236, - /* 1340 */ 175, 240, 217, 175, 180, 175, 164, 175, 187, 240, - /* 1350 */ 176, 164, 240, 240, 240, 240, 240, 240, 230, 207, - /* 1360 */ 240, 240, 240, 172, 240, 240, 240, 240, 172, 240, - /* 1370 */ 240, 240, 197, 181, 240, 181, 181, 230, 240, 194, - /* 1380 */ 197, 193, 189, 184, 240, 184, 207, 240, 207, 240, - /* 1390 */ 240, 240, 206, 240, 240, 240, 240, 240, 240, 240, - /* 1400 */ 240, 240, 240, 240, 240, 240, 239, 240, 240, 240, - /* 1410 */ 240, 240, 240, 240, 240, 240, 240, 239, 240, 240, - /* 1420 */ 240, 240, 233, 240, 234, + /* 950 */ 23, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 960 */ 91, 92, 19, 20, 148, 22, 148, 22, 148, 26, + /* 970 */ 27, 22, 148, 52, 1, 2, 22, 102, 19, 163, + /* 980 */ 37, 163, 148, 163, 20, 19, 20, 163, 22, 148, + /* 990 */ 172, 173, 26, 27, 109, 148, 24, 163, 104, 56, + /* 1000 */ 53, 43, 148, 37, 163, 53, 172, 173, 25, 66, + /* 1010 */ 163, 191, 148, 172, 173, 191, 138, 163, 148, 148, + /* 1020 */ 148, 59, 56, 104, 148, 148, 22, 163, 5, 86, + /* 1030 */ 87, 88, 66, 163, 163, 163, 93, 94, 95, 163, + /* 1040 */ 163, 98, 172, 173, 172, 173, 1, 118, 172, 173, + /* 1050 */ 148, 35, 86, 87, 148, 191, 148, 27, 108, 93, + /* 1060 */ 94, 95, 191, 76, 98, 163, 76, 119, 191, 163, + /* 1070 */ 22, 163, 129, 130, 131, 132, 133, 134, 172, 173, + /* 1080 */ 172, 173, 116, 127, 19, 20, 148, 22, 148, 148, + /* 1090 */ 20, 26, 27, 191, 148, 129, 130, 131, 132, 133, + /* 1100 */ 134, 163, 37, 163, 163, 148, 1, 19, 20, 163, + /* 1110 */ 22, 148, 172, 173, 26, 27, 119, 108, 172, 173, + /* 1120 */ 163, 56, 127, 22, 128, 37, 163, 3, 148, 172, + /* 1130 */ 173, 66, 191, 4, 148, 172, 173, 162, 178, 178, + /* 1140 */ 148, 148, 148, 163, 56, 6, 148, 147, 147, 163, + /* 1150 */ 147, 86, 87, 148, 66, 163, 163, 163, 93, 94, + /* 1160 */ 95, 163, 147, 98, 172, 173, 172, 173, 163, 148, + /* 1170 */ 172, 173, 13, 148, 86, 87, 148, 172, 173, 157, + /* 1180 */ 149, 93, 94, 95, 163, 148, 98, 148, 163, 148, + /* 1190 */ 148, 163, 148, 25, 129, 130, 131, 132, 133, 134, + /* 1200 */ 163, 148, 163, 149, 163, 163, 192, 163, 148, 172, + /* 1210 */ 173, 172, 173, 116, 172, 173, 163, 129, 130, 131, + /* 1220 */ 132, 133, 134, 163, 148, 172, 173, 148, 148, 148, + /* 1230 */ 148, 148, 172, 173, 126, 148, 198, 148, 148, 163, + /* 1240 */ 148, 148, 163, 163, 163, 163, 163, 148, 172, 173, + /* 1250 */ 163, 148, 163, 163, 148, 163, 163, 148, 148, 197, + /* 1260 */ 123, 200, 163, 199, 124, 122, 163, 201, 135, 163, + /* 1270 */ 125, 224, 163, 163, 155, 118, 155, 208, 104, 121, + /* 1280 */ 209, 208, 104, 209, 209, 208, 174, 174, 47, 182, + /* 1290 */ 174, 176, 103, 22, 174, 177, 92, 174, 154, 227, + /* 1300 */ 174, 174, 177, 18, 182, 227, 155, 154, 45, 155, + /* 1310 */ 154, 135, 68, 217, 22, 155, 155, 154, 234, 155, + /* 1320 */ 235, 187, 187, 18, 151, 197, 190, 197, 190, 164, + /* 1330 */ 175, 190, 190, 187, 217, 164, 238, 175, 238, 207, + /* 1340 */ 229, 194, 175, 229, 196, 175, 180, 175, 164, 197, + /* 1350 */ 164, 176, 146, 92, 239, 206, 225, 207, 239, 239, + /* 1360 */ 239, 239, 172, 172, 239, 239, 181, 239, 181, 181, + /* 1370 */ 239, 239, 189, 194, 197, 193, 207, 184, 184, 239, + /* 1380 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 1390 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 1400 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 1410 */ 239, 239, 232, 239, 233, }; -#define YY_SHIFT_USE_DFLT (-70) +#define YY_SHIFT_USE_DFLT (-93) #define YY_SHIFT_COUNT (373) -#define YY_SHIFT_MIN (-69) -#define YY_SHIFT_MAX (1301) +#define YY_SHIFT_MIN (-92) +#define YY_SHIFT_MAX (1305) static const short yy_shift_ofst[] = { - /* 0 */ 993, 966, 375, 966, 1065, 1065, 0, -19, 1065, 1065, - /* 10 */ 1065, 1065, 1065, 346, 316, 943, 1065, 1065, 1065, 1065, + /* 0 */ 973, 966, 375, 966, 1065, 1065, 0, -19, 1065, 1065, + /* 10 */ 1065, 1065, 1065, 346, 576, 943, 1065, 1065, 1065, 1065, /* 20 */ 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, /* 30 */ 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, /* 40 */ 1088, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, 1065, - /* 50 */ 1065, 1065, 1065, 1065, -49, 212, 890, 890, 432, 1145, - /* 60 */ 55, 573, 499, 425, 351, 277, 203, 129, 647, 647, + /* 50 */ 1065, 1065, 1065, 1065, 212, 840, 840, 162, 1097, 55, + /* 60 */ 573, 499, 425, 351, 277, 203, 129, 647, 647, 647, /* 70 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, - /* 80 */ 647, 647, 647, 647, 721, 647, 795, 869, 869, -69, - /* 90 */ -69, -69, -69, -1, -1, 58, 27, -44, 316, 316, - /* 100 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, - /* 110 */ 316, 316, 316, 316, 508, 316, 316, 316, 316, 316, - /* 120 */ 206, 432, 883, 1236, -70, -70, -70, 390, 73, 581, - /* 130 */ 581, 296, 282, 238, 221, 208, 316, 316, 316, 316, - /* 140 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, - /* 150 */ 316, 316, 316, 316, 316, 316, 316, 316, 316, 316, - /* 160 */ 316, 648, 653, 653, 653, 599, 435, 1145, 1145, 1145, - /* 170 */ -70, -70, 144, 95, 95, 84, 421, 421, 421, 370, - /* 180 */ 483, 513, 431, 424, 349, -36, -36, -36, -36, 566, - /* 190 */ 494, -36, -36, 372, 356, 145, 526, 498, 589, 589, - /* 200 */ 805, 498, 805, 453, 432, 79, 432, 79, 738, 79, - /* 210 */ 589, 79, 79, 715, 620, 620, 432, 299, 75, 797, - /* 220 */ 1292, 1243, 1136, 1301, 1301, 1301, 1301, 1136, 1292, 1243, - /* 230 */ 1243, 1156, 1276, 1177, 1263, 1156, 1156, 1276, 1156, 1276, - /* 240 */ 1156, 1276, 1264, 1176, 1176, 1176, 1235, 1204, 1204, 1264, - /* 250 */ 1176, 1180, 1176, 1235, 1176, 1176, 1157, 1173, 1157, 1173, - /* 260 */ 1157, 1173, 1156, 1156, 1133, 1146, 1143, 1142, 1141, 1136, - /* 270 */ 1145, 1216, 1226, 1226, 1209, 1209, 1209, 1209, -70, -70, - /* 280 */ -70, 482, 321, 624, 161, 749, 723, 684, 683, 460, - /* 290 */ 448, 313, 215, 21, 130, 443, 350, 1202, 1200, 1073, - /* 300 */ 1158, 1052, 1068, 1053, 1148, 1166, 1047, 1140, 1015, 1074, - /* 310 */ 1069, 1033, 1100, 1104, 1006, 1122, 1112, 1094, 1002, 952, - /* 320 */ 1017, 1043, 1013, 988, 1014, 947, 1022, 1008, 1004, 901, - /* 330 */ 791, 983, 930, 956, 949, 945, 893, 875, 752, 745, - /* 340 */ 726, 659, 887, 823, 592, 734, 634, 544, 501, 519, - /* 350 */ 512, 525, 380, 519, 464, 426, 505, 423, 382, 320, - /* 360 */ 303, 284, 261, 248, 23, 141, 138, 78, 46, -25, - /* 370 */ 40, 2, -3, 5, + /* 80 */ 647, 647, 647, 721, 647, 795, 869, 869, -69, -45, + /* 90 */ -45, -45, -45, -45, -1, 58, 27, 305, 576, 576, + /* 100 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, + /* 110 */ 576, 576, 576, 576, 586, 576, 576, 576, 576, 576, + /* 120 */ 466, 162, 811, 1261, -93, -93, -93, 390, 73, 581, + /* 130 */ 581, 296, 282, 238, 221, 208, 576, 576, 576, 576, + /* 140 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, + /* 150 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, + /* 160 */ 576, 582, 312, 312, 312, 619, -92, 1097, 1097, 1097, + /* 170 */ -93, -93, 144, 95, 95, 84, 427, 427, 427, 504, + /* 180 */ 500, 585, 444, 431, 290, -36, -36, -36, -36, 366, + /* 190 */ 494, -36, -36, 443, 356, 145, 335, 514, 430, 430, + /* 200 */ 506, 514, 506, 454, 162, 79, 162, 79, 275, 79, + /* 210 */ 430, 79, 79, 502, 99, 99, 162, 299, 125, -18, + /* 220 */ 1292, 1244, 1108, 1305, 1305, 1305, 1305, 1108, 1292, 1244, + /* 230 */ 1244, 1157, 1285, 1176, 1263, 1157, 1157, 1285, 1157, 1285, + /* 240 */ 1157, 1285, 1271, 1178, 1178, 1178, 1241, 1204, 1204, 1271, + /* 250 */ 1178, 1189, 1178, 1241, 1178, 1178, 1158, 1174, 1158, 1174, + /* 260 */ 1158, 1174, 1157, 1157, 1133, 1145, 1143, 1140, 1137, 1108, + /* 270 */ 1097, 1168, 1159, 1159, 1139, 1139, 1139, 1139, -93, -93, + /* 280 */ -93, 482, 321, 179, 161, 826, 719, 653, 605, 583, + /* 290 */ 575, 511, 54, 21, 300, 467, 350, 1129, 1124, 996, + /* 300 */ 1101, 995, 1009, 997, 1070, 1105, 948, 1048, 956, 990, + /* 310 */ 987, 950, 1030, 1016, 929, 1045, 1023, 1004, 919, 878, + /* 320 */ 952, 983, 947, 962, 958, 894, 972, 964, 959, 885, + /* 330 */ 875, 954, 921, 949, 945, 927, 916, 884, 899, 909, + /* 340 */ 863, 860, 839, 836, 778, 825, 806, 814, 730, 689, + /* 350 */ 740, 739, 716, 689, 734, 678, 669, 609, 525, 584, + /* 360 */ 522, 507, 483, 436, 302, 254, 194, 166, 152, 141, + /* 370 */ 174, 113, 78, 47, }; -#define YY_REDUCE_USE_DFLT (-220) +#define YY_REDUCE_USE_DFLT (-167) #define YY_REDUCE_COUNT (280) -#define YY_REDUCE_MIN (-219) -#define YY_REDUCE_MAX (1201) +#define YY_REDUCE_MIN (-166) +#define YY_REDUCE_MAX (1206) static const short yy_reduce_ofst[] = { - /* 0 */ 490, 18, 105, 297, 88, -51, 151, 176, 456, 392, - /* 10 */ 157, 389, 14, 402, 246, 998, 992, 984, 963, 957, - /* 20 */ 946, 940, 908, 906, 872, 870, 852, 841, 835, 824, - /* 30 */ 818, 763, 760, 757, 744, 741, 728, 688, 686, 675, - /* 40 */ 667, 654, 614, 612, 610, 596, 593, 580, 541, 539, - /* 50 */ 528, 522, 520, 463, -219, 462, 446, 438, -58, 37, - /* 60 */ -71, -71, -71, -71, -71, -71, -71, -71, -71, -71, - /* 70 */ -71, -71, -71, -71, -71, -71, -71, -71, -71, -71, - /* 80 */ -71, -71, -71, -71, -71, -71, -71, -71, -71, -71, - /* 90 */ -71, -71, -71, -71, -71, 584, -71, -71, 1037, 680, - /* 100 */ 1025, 1021, 986, 902, 892, 871, 774, 980, 674, 83, - /* 110 */ 542, 854, 816, 858, 228, 746, 672, 668, 594, 540, - /* 120 */ -129, 516, -71, -71, -71, -71, -71, 895, 662, 736, - /* 130 */ 714, 941, 1092, 1090, 1089, 1087, 1086, 1085, 1084, 1083, - /* 140 */ 1082, 1079, 1066, 1063, 1062, 1061, 1060, 1044, 1042, 1041, - /* 150 */ 1039, 1030, 990, 985, 974, 941, 938, 900, 863, 853, - /* 160 */ 758, 792, 770, 764, 735, 755, 704, 698, 671, 445, - /* 170 */ 650, 623, 1201, 1196, 1191, 1199, 1195, 1194, 1192, 1186, - /* 180 */ 1188, 1193, 1188, 1188, 1188, 1188, 1188, 1188, 1188, 1190, - /* 190 */ 1189, 1188, 1188, 1186, 1187, 1185, 1184, 1183, 1181, 1179, - /* 200 */ 1178, 1175, 1167, 1174, 1182, 1172, 1171, 1170, 1164, 1168, - /* 210 */ 1152, 1165, 1162, 1106, 1147, 1128, 1163, 1137, 1144, 1169, - /* 220 */ 1125, 1161, 1139, 1135, 1134, 1132, 1131, 1121, 1114, 1130, - /* 230 */ 1129, 1160, 1159, 1103, 1099, 1155, 1154, 1153, 1151, 1149, - /* 240 */ 1150, 1138, 1127, 1126, 1124, 1123, 1119, 1098, 1095, 1116, - /* 250 */ 1117, 1108, 1115, 1113, 1107, 1105, 1093, 1091, 1081, 1080, - /* 260 */ 1078, 1077, 1120, 1118, 1045, 1075, 1072, 1064, 1071, 1070, - /* 270 */ 1067, 1101, 1111, 1102, 1110, 1109, 1097, 1096, 1076, 1058, - /* 280 */ 1051, + /* 0 */ 532, 18, 105, 297, 88, -51, 151, 244, 392, 389, + /* 10 */ 157, 14, 378, 475, 251, 1076, 1060, 1053, 1042, 1039, + /* 20 */ 1037, 1005, 998, 994, 992, 963, 957, 946, 940, 908, + /* 30 */ 906, 876, 872, 870, 841, 834, 818, 762, 760, 757, + /* 40 */ 749, 744, 741, 728, 670, 667, 654, 613, 611, 606, + /* 50 */ 600, 593, 580, 536, 477, 449, -166, -58, 476, 465, + /* 60 */ 465, 465, 465, 465, 465, 465, 465, 465, 465, 465, + /* 70 */ 465, 465, 465, 465, 465, 465, 465, 465, 465, 465, + /* 80 */ 465, 465, 465, 465, 465, 465, 465, 465, 465, 465, + /* 90 */ 465, 465, 465, 465, 465, 579, 465, 465, 824, 675, + /* 100 */ 941, 902, 877, 871, 864, 820, 700, 684, 674, 83, + /* 110 */ 463, 672, 614, 1, 503, 671, 668, 597, 298, 544, + /* 120 */ -128, 538, 465, 465, 465, 465, 465, 168, 727, 640, + /* 130 */ 599, 938, 1110, 1109, 1106, 1103, 1099, 1093, 1092, 1090, + /* 140 */ 1089, 1087, 1083, 1082, 1081, 1080, 1079, 1044, 1041, 1028, + /* 150 */ 1025, 1021, 993, 986, 980, 938, 854, 847, 816, 761, + /* 160 */ 698, 666, 691, 690, 657, 625, 589, 588, 496, 402, + /* 170 */ 102, 510, 1194, 1191, 1190, 1193, 1188, 1187, 1185, 1149, + /* 180 */ 1182, 1183, 1182, 1182, 1182, 1182, 1182, 1182, 1182, 1181, + /* 190 */ 1180, 1182, 1182, 1149, 1186, 1179, 1206, 1177, 1169, 1150, + /* 200 */ 1100, 1152, 1098, 1175, 1184, 1172, 1171, 1170, 1166, 1167, + /* 210 */ 1132, 1162, 1155, 1131, 1114, 1111, 1165, 1148, 1147, 1173, + /* 220 */ 1117, 1146, 1130, 1142, 1141, 1138, 1136, 1128, 1096, 1135, + /* 230 */ 1134, 1164, 1163, 1085, 1084, 1161, 1160, 1156, 1154, 1153, + /* 240 */ 1151, 1144, 1125, 1127, 1126, 1123, 1122, 1078, 1072, 1118, + /* 250 */ 1120, 1115, 1116, 1107, 1113, 1112, 1075, 1077, 1074, 1073, + /* 260 */ 1071, 1069, 1121, 1119, 1047, 1066, 1061, 1064, 1038, 1062, + /* 270 */ 1014, 1022, 1054, 1031, 1015, 1003, 1001, 1000, 961, 960, + /* 280 */ 975, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 566, 802, 851, 851, 851, 802, 851, 690, 851, 851, - /* 10 */ 851, 851, 800, 851, 851, 851, 851, 851, 851, 851, - /* 20 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 30 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 40 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 50 */ 851, 851, 851, 851, 774, 851, 819, 819, 605, 694, - /* 60 */ 725, 851, 851, 851, 851, 851, 851, 851, 740, 739, - /* 70 */ 733, 732, 832, 705, 730, 723, 716, 727, 803, 796, - /* 80 */ 797, 795, 799, 804, 851, 726, 762, 780, 761, 779, - /* 90 */ 786, 778, 764, 773, 763, 597, 765, 766, 851, 851, - /* 100 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 110 */ 851, 851, 851, 851, 659, 851, 851, 851, 851, 851, - /* 120 */ 592, 851, 767, 768, 783, 782, 781, 851, 851, 851, - /* 130 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 140 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 150 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 160 */ 572, 851, 690, 690, 690, 566, 851, 851, 851, 851, - /* 170 */ 694, 684, 650, 851, 851, 851, 851, 851, 851, 851, - /* 180 */ 851, 851, 851, 851, 851, 735, 673, 842, 844, 851, - /* 190 */ 825, 671, 594, 692, 607, 682, 574, 729, 707, 707, - /* 200 */ 837, 729, 837, 631, 851, 719, 851, 719, 628, 719, - /* 210 */ 707, 719, 719, 798, 851, 851, 851, 691, 682, 851, - /* 220 */ 741, 663, 729, 670, 670, 670, 670, 729, 741, 663, - /* 230 */ 663, 698, 589, 831, 829, 698, 698, 589, 698, 589, - /* 240 */ 698, 589, 808, 661, 661, 661, 646, 812, 812, 808, - /* 250 */ 661, 631, 661, 646, 661, 661, 711, 706, 711, 706, - /* 260 */ 711, 706, 698, 698, 851, 724, 712, 722, 720, 729, - /* 270 */ 851, 649, 582, 582, 571, 571, 571, 571, 633, 633, - /* 280 */ 615, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 290 */ 851, 851, 851, 851, 851, 851, 851, 851, 567, 851, - /* 300 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 310 */ 851, 851, 851, 851, 835, 851, 851, 851, 851, 851, - /* 320 */ 851, 828, 827, 851, 851, 851, 851, 851, 851, 851, - /* 330 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 851, - /* 340 */ 851, 851, 851, 851, 851, 851, 851, 851, 851, 721, - /* 350 */ 851, 713, 851, 801, 851, 851, 851, 851, 851, 851, - /* 360 */ 851, 851, 851, 851, 676, 750, 851, 749, 753, 748, - /* 370 */ 599, 851, 580, 851, 563, 568, 736, 734, 731, 728, - /* 380 */ 850, 814, 672, 669, 668, 588, 834, 843, 841, 742, - /* 390 */ 840, 839, 838, 836, 833, 820, 738, 737, 664, 806, - /* 400 */ 805, 591, 824, 823, 822, 826, 830, 821, 700, 590, - /* 410 */ 587, 596, 653, 652, 660, 658, 657, 656, 655, 654, - /* 420 */ 651, 598, 606, 617, 645, 630, 629, 811, 813, 810, - /* 430 */ 809, 638, 637, 643, 642, 641, 640, 639, 636, 635, - /* 440 */ 634, 627, 626, 632, 625, 648, 647, 644, 624, 667, - /* 450 */ 666, 665, 662, 623, 622, 621, 753, 620, 619, 759, - /* 460 */ 758, 746, 790, 687, 686, 685, 697, 696, 709, 708, - /* 470 */ 744, 743, 710, 695, 689, 688, 704, 703, 702, 701, - /* 480 */ 693, 683, 715, 718, 717, 714, 775, 792, 699, 789, - /* 490 */ 849, 848, 847, 846, 845, 794, 793, 760, 757, 610, - /* 500 */ 611, 818, 816, 817, 815, 613, 612, 609, 608, 791, - /* 510 */ 678, 677, 787, 784, 776, 771, 788, 785, 777, 772, - /* 520 */ 770, 769, 755, 754, 752, 751, 747, 756, 601, 679, - /* 530 */ 675, 674, 745, 681, 680, 618, 616, 614, 595, 593, - /* 540 */ 586, 584, 583, 585, 581, 579, 578, 577, 576, 575, - /* 550 */ 604, 603, 602, 600, 599, 573, 570, 569, 565, 564, - /* 560 */ 562, + /* 0 */ 565, 800, 849, 849, 849, 800, 849, 689, 849, 849, + /* 10 */ 849, 849, 798, 849, 849, 849, 849, 849, 849, 849, + /* 20 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 30 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 40 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 50 */ 849, 849, 849, 849, 849, 817, 817, 604, 693, 724, + /* 60 */ 849, 849, 849, 849, 849, 849, 849, 739, 738, 732, + /* 70 */ 731, 830, 704, 729, 722, 715, 726, 801, 794, 795, + /* 80 */ 793, 797, 802, 849, 725, 761, 778, 760, 772, 777, + /* 90 */ 784, 776, 773, 763, 762, 596, 764, 765, 849, 849, + /* 100 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 110 */ 849, 849, 849, 849, 658, 849, 849, 849, 849, 849, + /* 120 */ 591, 849, 766, 767, 781, 780, 779, 849, 849, 849, + /* 130 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 140 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 150 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 160 */ 571, 849, 689, 689, 689, 565, 849, 849, 849, 849, + /* 170 */ 693, 683, 649, 849, 849, 849, 849, 849, 849, 849, + /* 180 */ 849, 849, 849, 849, 849, 734, 672, 840, 842, 849, + /* 190 */ 823, 670, 593, 691, 606, 681, 573, 728, 706, 706, + /* 200 */ 835, 728, 835, 630, 849, 718, 849, 718, 627, 718, + /* 210 */ 706, 718, 718, 796, 849, 849, 849, 690, 681, 849, + /* 220 */ 740, 662, 728, 669, 669, 669, 669, 728, 740, 662, + /* 230 */ 662, 697, 588, 829, 827, 697, 697, 588, 697, 588, + /* 240 */ 697, 588, 806, 660, 660, 660, 645, 810, 810, 806, + /* 250 */ 660, 630, 660, 645, 660, 660, 710, 705, 710, 705, + /* 260 */ 710, 705, 697, 697, 849, 723, 711, 721, 719, 728, + /* 270 */ 849, 648, 581, 581, 570, 570, 570, 570, 632, 632, + /* 280 */ 614, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 290 */ 849, 849, 849, 849, 849, 849, 849, 849, 566, 849, + /* 300 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 310 */ 849, 849, 849, 849, 833, 849, 849, 849, 849, 849, + /* 320 */ 849, 826, 825, 849, 849, 849, 849, 849, 849, 849, + /* 330 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 849, + /* 340 */ 849, 849, 849, 849, 849, 849, 849, 849, 849, 720, + /* 350 */ 849, 712, 849, 799, 849, 849, 849, 849, 849, 849, + /* 360 */ 849, 849, 849, 849, 675, 749, 849, 748, 752, 747, + /* 370 */ 598, 849, 579, 849, 562, 567, 735, 733, 730, 727, + /* 380 */ 848, 812, 671, 668, 667, 587, 832, 841, 839, 741, + /* 390 */ 838, 837, 836, 834, 831, 818, 737, 736, 663, 804, + /* 400 */ 803, 590, 822, 821, 820, 824, 828, 819, 699, 589, + /* 410 */ 586, 595, 652, 651, 659, 657, 656, 655, 654, 653, + /* 420 */ 650, 597, 605, 616, 644, 629, 628, 809, 811, 808, + /* 430 */ 807, 637, 636, 642, 641, 640, 639, 638, 635, 634, + /* 440 */ 633, 626, 625, 631, 624, 647, 646, 643, 623, 666, + /* 450 */ 665, 664, 661, 622, 621, 620, 752, 619, 618, 758, + /* 460 */ 757, 745, 788, 686, 685, 684, 696, 695, 708, 707, + /* 470 */ 743, 742, 709, 694, 688, 687, 703, 702, 701, 700, + /* 480 */ 692, 682, 714, 717, 716, 713, 790, 698, 787, 847, + /* 490 */ 846, 845, 844, 843, 792, 791, 759, 756, 609, 610, + /* 500 */ 816, 814, 815, 813, 612, 611, 608, 607, 789, 677, + /* 510 */ 676, 785, 782, 774, 770, 786, 783, 775, 771, 769, + /* 520 */ 768, 754, 753, 751, 750, 746, 755, 600, 678, 674, + /* 530 */ 673, 744, 680, 679, 617, 615, 613, 594, 592, 585, + /* 540 */ 583, 582, 584, 580, 578, 577, 576, 575, 574, 603, + /* 550 */ 602, 601, 599, 598, 572, 569, 568, 564, 563, 561, }; /* The next table maps tokens into fallback tokens. If a construct ** like the following: ** @@ -98458,15 +100953,15 @@ "orderby_opt", "limit_opt", "sclp", "as", "seltablist", "stl_prefix", "joinop", "indexed_opt", "on_opt", "using_opt", "joinop2", "inscollist", "sortlist", "sortitem", "nexprlist", "setlist", "insert_cmd", "inscollist_opt", "itemlist", "exprlist", - "likeop", "escape", "between_op", "in_op", - "case_operand", "case_exprlist", "case_else", "uniqueflag", - "collate", "plus_opt", "number", "trigger_decl", - "trigger_cmd_list", "trigger_time", "trigger_event", "foreach_clause", - "when_clause", "trigger_cmd", "trnm", "tridxby", + "likeop", "between_op", "in_op", "case_operand", + "case_exprlist", "case_else", "uniqueflag", "collate", + "plus_opt", "number", "trigger_decl", "trigger_cmd_list", + "trigger_time", "trigger_event", "foreach_clause", "when_clause", + "trigger_cmd", "trnm", "tridxby", }; #endif /* NDEBUG */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. @@ -98682,88 +101177,87 @@ /* 207 */ "expr ::= expr CONCAT expr", /* 208 */ "likeop ::= LIKE_KW", /* 209 */ "likeop ::= NOT LIKE_KW", /* 210 */ "likeop ::= MATCH", /* 211 */ "likeop ::= NOT MATCH", - /* 212 */ "escape ::= ESCAPE expr", - /* 213 */ "escape ::=", - /* 214 */ "expr ::= expr likeop expr escape", - /* 215 */ "expr ::= expr ISNULL|NOTNULL", - /* 216 */ "expr ::= expr NOT NULL", - /* 217 */ "expr ::= expr IS expr", - /* 218 */ "expr ::= expr IS NOT expr", - /* 219 */ "expr ::= NOT expr", - /* 220 */ "expr ::= BITNOT expr", - /* 221 */ "expr ::= MINUS expr", - /* 222 */ "expr ::= PLUS expr", - /* 223 */ "between_op ::= BETWEEN", - /* 224 */ "between_op ::= NOT BETWEEN", - /* 225 */ "expr ::= expr between_op expr AND expr", - /* 226 */ "in_op ::= IN", - /* 227 */ "in_op ::= NOT IN", - /* 228 */ "expr ::= expr in_op LP exprlist RP", - /* 229 */ "expr ::= LP select RP", - /* 230 */ "expr ::= expr in_op LP select RP", - /* 231 */ "expr ::= expr in_op nm dbnm", - /* 232 */ "expr ::= EXISTS LP select RP", - /* 233 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 234 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 235 */ "case_exprlist ::= WHEN expr THEN expr", - /* 236 */ "case_else ::= ELSE expr", - /* 237 */ "case_else ::=", - /* 238 */ "case_operand ::= expr", - /* 239 */ "case_operand ::=", - /* 240 */ "exprlist ::= nexprlist", - /* 241 */ "exprlist ::=", - /* 242 */ "nexprlist ::= nexprlist COMMA expr", - /* 243 */ "nexprlist ::= expr", - /* 244 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP", - /* 245 */ "uniqueflag ::= UNIQUE", - /* 246 */ "uniqueflag ::=", - /* 247 */ "idxlist_opt ::=", - /* 248 */ "idxlist_opt ::= LP idxlist RP", - /* 249 */ "idxlist ::= idxlist COMMA nm collate sortorder", - /* 250 */ "idxlist ::= nm collate sortorder", - /* 251 */ "collate ::=", - /* 252 */ "collate ::= COLLATE ids", - /* 253 */ "cmd ::= DROP INDEX ifexists fullname", - /* 254 */ "plus_num ::= plus_opt number", - /* 255 */ "minus_num ::= MINUS number", - /* 256 */ "number ::= INTEGER|FLOAT", - /* 257 */ "plus_opt ::= PLUS", - /* 258 */ "plus_opt ::=", - /* 259 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 260 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 261 */ "trigger_time ::= BEFORE", - /* 262 */ "trigger_time ::= AFTER", - /* 263 */ "trigger_time ::= INSTEAD OF", - /* 264 */ "trigger_time ::=", - /* 265 */ "trigger_event ::= DELETE|INSERT", - /* 266 */ "trigger_event ::= UPDATE", - /* 267 */ "trigger_event ::= UPDATE OF inscollist", - /* 268 */ "foreach_clause ::=", - /* 269 */ "foreach_clause ::= FOR EACH ROW", - /* 270 */ "when_clause ::=", - /* 271 */ "when_clause ::= WHEN expr", - /* 272 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 273 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 274 */ "trnm ::= nm", - /* 275 */ "trnm ::= nm DOT nm", - /* 276 */ "tridxby ::=", - /* 277 */ "tridxby ::= INDEXED BY nm", - /* 278 */ "tridxby ::= NOT INDEXED", - /* 279 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 280 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP", - /* 281 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", - /* 282 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 283 */ "trigger_cmd ::= select", - /* 284 */ "expr ::= RAISE LP IGNORE RP", - /* 285 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 286 */ "raisetype ::= ROLLBACK", - /* 287 */ "raisetype ::= ABORT", - /* 288 */ "raisetype ::= FAIL", - /* 289 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 212 */ "expr ::= expr likeop expr", + /* 213 */ "expr ::= expr likeop expr ESCAPE expr", + /* 214 */ "expr ::= expr ISNULL|NOTNULL", + /* 215 */ "expr ::= expr NOT NULL", + /* 216 */ "expr ::= expr IS expr", + /* 217 */ "expr ::= expr IS NOT expr", + /* 218 */ "expr ::= NOT expr", + /* 219 */ "expr ::= BITNOT expr", + /* 220 */ "expr ::= MINUS expr", + /* 221 */ "expr ::= PLUS expr", + /* 222 */ "between_op ::= BETWEEN", + /* 223 */ "between_op ::= NOT BETWEEN", + /* 224 */ "expr ::= expr between_op expr AND expr", + /* 225 */ "in_op ::= IN", + /* 226 */ "in_op ::= NOT IN", + /* 227 */ "expr ::= expr in_op LP exprlist RP", + /* 228 */ "expr ::= LP select RP", + /* 229 */ "expr ::= expr in_op LP select RP", + /* 230 */ "expr ::= expr in_op nm dbnm", + /* 231 */ "expr ::= EXISTS LP select RP", + /* 232 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 233 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 234 */ "case_exprlist ::= WHEN expr THEN expr", + /* 235 */ "case_else ::= ELSE expr", + /* 236 */ "case_else ::=", + /* 237 */ "case_operand ::= expr", + /* 238 */ "case_operand ::=", + /* 239 */ "exprlist ::= nexprlist", + /* 240 */ "exprlist ::=", + /* 241 */ "nexprlist ::= nexprlist COMMA expr", + /* 242 */ "nexprlist ::= expr", + /* 243 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP", + /* 244 */ "uniqueflag ::= UNIQUE", + /* 245 */ "uniqueflag ::=", + /* 246 */ "idxlist_opt ::=", + /* 247 */ "idxlist_opt ::= LP idxlist RP", + /* 248 */ "idxlist ::= idxlist COMMA nm collate sortorder", + /* 249 */ "idxlist ::= nm collate sortorder", + /* 250 */ "collate ::=", + /* 251 */ "collate ::= COLLATE ids", + /* 252 */ "cmd ::= DROP INDEX ifexists fullname", + /* 253 */ "plus_num ::= plus_opt number", + /* 254 */ "minus_num ::= MINUS number", + /* 255 */ "number ::= INTEGER|FLOAT", + /* 256 */ "plus_opt ::= PLUS", + /* 257 */ "plus_opt ::=", + /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 260 */ "trigger_time ::= BEFORE", + /* 261 */ "trigger_time ::= AFTER", + /* 262 */ "trigger_time ::= INSTEAD OF", + /* 263 */ "trigger_time ::=", + /* 264 */ "trigger_event ::= DELETE|INSERT", + /* 265 */ "trigger_event ::= UPDATE", + /* 266 */ "trigger_event ::= UPDATE OF inscollist", + /* 267 */ "foreach_clause ::=", + /* 268 */ "foreach_clause ::= FOR EACH ROW", + /* 269 */ "when_clause ::=", + /* 270 */ "when_clause ::= WHEN expr", + /* 271 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 272 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 273 */ "trnm ::= nm", + /* 274 */ "trnm ::= nm DOT nm", + /* 275 */ "tridxby ::=", + /* 276 */ "tridxby ::= INDEXED BY nm", + /* 277 */ "tridxby ::= NOT INDEXED", + /* 278 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", + /* 279 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP", + /* 280 */ "trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select", + /* 281 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", + /* 282 */ "trigger_cmd ::= select", + /* 283 */ "expr ::= RAISE LP IGNORE RP", + /* 284 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 285 */ "raisetype ::= ROLLBACK", + /* 286 */ "raisetype ::= ABORT", + /* 287 */ "raisetype ::= FAIL", + /* 288 */ "cmd ::= DROP TRIGGER ifexists fullname", }; #endif /* NDEBUG */ #if YYSTACKDEPTH<=0 @@ -98841,18 +101335,17 @@ ** inside the C code. */ case 158: /* select */ case 192: /* oneselect */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy459)); +sqlite3SelectDelete(pParse->db, (yypminor->yy239)); } break; case 172: /* term */ case 173: /* expr */ - case 221: /* escape */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy22).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy64).pExpr); } break; case 177: /* idxlist_opt */ case 185: /* idxlist */ case 195: /* selcollist */ @@ -98862,50 +101355,50 @@ case 212: /* sortlist */ case 214: /* nexprlist */ case 215: /* setlist */ case 218: /* itemlist */ case 219: /* exprlist */ - case 225: /* case_exprlist */ + case 224: /* case_exprlist */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy82)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy462)); } break; case 191: /* fullname */ case 196: /* from */ case 204: /* seltablist */ case 205: /* stl_prefix */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy67)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy285)); } break; case 197: /* where_opt */ case 199: /* having_opt */ case 208: /* on_opt */ case 213: /* sortitem */ - case 224: /* case_operand */ - case 226: /* case_else */ - case 236: /* when_clause */ + case 223: /* case_operand */ + case 225: /* case_else */ + case 235: /* when_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy2)); +sqlite3ExprDelete(pParse->db, (yypminor->yy178)); } break; case 209: /* using_opt */ case 211: /* inscollist */ case 217: /* inscollist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy240)); +sqlite3IdListDelete(pParse->db, (yypminor->yy160)); } break; - case 232: /* trigger_cmd_list */ - case 237: /* trigger_cmd */ + case 231: /* trigger_cmd_list */ + case 236: /* trigger_cmd */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy347)); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy247)); } break; - case 234: /* trigger_event */ + case 233: /* trigger_event */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy210).b); +sqlite3IdListDelete(pParse->db, (yypminor->yy132).b); } break; default: break; /* If no destructor action specified: do nothing */ } } @@ -99359,82 +101852,81 @@ { 173, 3 }, { 220, 1 }, { 220, 2 }, { 220, 1 }, { 220, 2 }, - { 221, 2 }, - { 221, 0 }, - { 173, 4 }, + { 173, 3 }, + { 173, 5 }, { 173, 2 }, { 173, 3 }, { 173, 3 }, { 173, 4 }, { 173, 2 }, { 173, 2 }, { 173, 2 }, { 173, 2 }, + { 221, 1 }, + { 221, 2 }, + { 173, 5 }, { 222, 1 }, { 222, 2 }, - { 173, 5 }, - { 223, 1 }, - { 223, 2 }, { 173, 5 }, { 173, 3 }, { 173, 5 }, { 173, 4 }, { 173, 4 }, { 173, 5 }, - { 225, 5 }, - { 225, 4 }, - { 226, 2 }, - { 226, 0 }, - { 224, 1 }, - { 224, 0 }, + { 224, 5 }, + { 224, 4 }, + { 225, 2 }, + { 225, 0 }, + { 223, 1 }, + { 223, 0 }, { 219, 1 }, { 219, 0 }, { 214, 3 }, { 214, 1 }, { 145, 11 }, - { 227, 1 }, - { 227, 0 }, + { 226, 1 }, + { 226, 0 }, { 177, 0 }, { 177, 3 }, { 185, 5 }, { 185, 3 }, - { 228, 0 }, - { 228, 2 }, + { 227, 0 }, + { 227, 2 }, { 145, 4 }, { 168, 2 }, { 169, 2 }, - { 230, 1 }, { 229, 1 }, - { 229, 0 }, + { 228, 1 }, + { 228, 0 }, { 145, 5 }, - { 231, 11 }, + { 230, 11 }, + { 232, 1 }, + { 232, 1 }, + { 232, 2 }, + { 232, 0 }, { 233, 1 }, { 233, 1 }, - { 233, 2 }, - { 233, 0 }, - { 234, 1 }, - { 234, 1 }, + { 233, 3 }, + { 234, 0 }, { 234, 3 }, { 235, 0 }, - { 235, 3 }, - { 236, 0 }, - { 236, 2 }, - { 232, 3 }, - { 232, 2 }, - { 238, 1 }, + { 235, 2 }, + { 231, 3 }, + { 231, 2 }, + { 237, 1 }, + { 237, 3 }, + { 238, 0 }, { 238, 3 }, - { 239, 0 }, - { 239, 3 }, - { 239, 2 }, - { 237, 7 }, - { 237, 8 }, - { 237, 5 }, - { 237, 5 }, - { 237, 1 }, + { 238, 2 }, + { 236, 7 }, + { 236, 8 }, + { 236, 5 }, + { 236, 5 }, + { 236, 1 }, { 173, 4 }, { 173, 6 }, { 189, 1 }, { 189, 1 }, { 189, 1 }, @@ -99504,21 +101996,21 @@ break; case 8: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } break; case 9: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy412);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy230);} break; case 13: /* transtype ::= */ -{yygotominor.yy412 = TK_DEFERRED;} +{yygotominor.yy230 = TK_DEFERRED;} break; case 14: /* transtype ::= DEFERRED */ case 15: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==15); case 16: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==16); case 115: /* multiselect_op ::= UNION */ yytestcase(yyruleno==115); case 117: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==117); -{yygotominor.yy412 = yymsp[0].major;} +{yygotominor.yy230 = yymsp[0].major;} break; case 17: /* cmd ::= COMMIT trans_opt */ case 18: /* cmd ::= END trans_opt */ yytestcase(yyruleno==18); {sqlite3CommitTransaction(pParse);} break; @@ -99540,11 +102032,11 @@ sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); } break; case 26: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy412,0,0,yymsp[-2].minor.yy412); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy230,0,0,yymsp[-2].minor.yy230); } break; case 27: /* createkw ::= CREATE */ { pParse->db->lookaside.bEnabled = 0; @@ -99559,33 +102051,33 @@ case 87: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ yytestcase(yyruleno==87); case 98: /* defer_subclause_opt ::= */ yytestcase(yyruleno==98); case 109: /* ifexists ::= */ yytestcase(yyruleno==109); case 120: /* distinct ::= ALL */ yytestcase(yyruleno==120); case 121: /* distinct ::= */ yytestcase(yyruleno==121); - case 223: /* between_op ::= BETWEEN */ yytestcase(yyruleno==223); - case 226: /* in_op ::= IN */ yytestcase(yyruleno==226); -{yygotominor.yy412 = 0;} + case 222: /* between_op ::= BETWEEN */ yytestcase(yyruleno==222); + case 225: /* in_op ::= IN */ yytestcase(yyruleno==225); +{yygotominor.yy230 = 0;} break; case 29: /* ifnotexists ::= IF NOT EXISTS */ case 30: /* temp ::= TEMP */ yytestcase(yyruleno==30); case 71: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==71); case 86: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ yytestcase(yyruleno==86); case 108: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==108); case 119: /* distinct ::= DISTINCT */ yytestcase(yyruleno==119); - case 224: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==224); - case 227: /* in_op ::= NOT IN */ yytestcase(yyruleno==227); -{yygotominor.yy412 = 1;} + case 223: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==223); + case 226: /* in_op ::= NOT IN */ yytestcase(yyruleno==226); +{yygotominor.yy230 = 1;} break; case 32: /* create_table_args ::= LP columnlist conslist_opt RP */ { sqlite3EndTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0); } break; case 33: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy459); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy459); + sqlite3EndTable(pParse,0,0,yymsp[0].minor.yy239); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy239); } break; case 36: /* column ::= columnid type carglist */ { yygotominor.yy0.z = yymsp[-2].minor.yy0.z; @@ -99608,15 +102100,15 @@ case 49: /* typename ::= ids */ yytestcase(yyruleno==49); case 127: /* as ::= AS nm */ yytestcase(yyruleno==127); case 128: /* as ::= ids */ yytestcase(yyruleno==128); case 138: /* dbnm ::= DOT nm */ yytestcase(yyruleno==138); case 147: /* indexed_opt ::= INDEXED BY nm */ yytestcase(yyruleno==147); - case 252: /* collate ::= COLLATE ids */ yytestcase(yyruleno==252); - case 254: /* plus_num ::= plus_opt number */ yytestcase(yyruleno==254); - case 255: /* minus_num ::= MINUS number */ yytestcase(yyruleno==255); - case 256: /* number ::= INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 274: /* trnm ::= nm */ yytestcase(yyruleno==274); + case 251: /* collate ::= COLLATE ids */ yytestcase(yyruleno==251); + case 253: /* plus_num ::= plus_opt number */ yytestcase(yyruleno==253); + case 254: /* minus_num ::= MINUS number */ yytestcase(yyruleno==254); + case 255: /* number ::= INTEGER|FLOAT */ yytestcase(yyruleno==255); + case 273: /* trnm ::= nm */ yytestcase(yyruleno==273); {yygotominor.yy0 = yymsp[0].minor.yy0;} break; case 45: /* type ::= typetoken */ {sqlite3AddColumnType(pParse,&yymsp[0].minor.yy0);} break; @@ -99635,21 +102127,21 @@ case 50: /* typename ::= typename ids */ {yygotominor.yy0.z=yymsp[-1].minor.yy0.z; yygotominor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} break; case 57: /* ccons ::= DEFAULT term */ case 59: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==59); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy22);} +{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy64);} break; case 58: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy22);} +{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy64);} break; case 60: /* ccons ::= DEFAULT MINUS term */ { ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy22.pExpr, 0, 0); + v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy64.pExpr, 0, 0); v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy22.zEnd; + v.zEnd = yymsp[0].minor.yy64.zEnd; sqlite3AddDefaultValue(pParse,&v); } break; case 61: /* ccons ::= DEFAULT id */ { @@ -99657,454 +102149,453 @@ spanExpr(&v, pParse, TK_STRING, &yymsp[0].minor.yy0); sqlite3AddDefaultValue(pParse,&v); } break; case 63: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy412);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy230);} break; case 64: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy412,yymsp[0].minor.yy412,yymsp[-2].minor.yy412);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy230,yymsp[0].minor.yy230,yymsp[-2].minor.yy230);} break; case 65: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy412,0,0,0,0);} +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy230,0,0,0,0);} break; case 66: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy22.pExpr);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy64.pExpr);} break; case 67: /* ccons ::= REFERENCES nm idxlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy82,yymsp[0].minor.yy412);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy462,yymsp[0].minor.yy230);} break; case 68: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy412);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy230);} break; case 69: /* ccons ::= COLLATE ids */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 72: /* refargs ::= */ -{ yygotominor.yy412 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yygotominor.yy230 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 73: /* refargs ::= refargs refarg */ -{ yygotominor.yy412 = (yymsp[-1].minor.yy412 & ~yymsp[0].minor.yy47.mask) | yymsp[0].minor.yy47.value; } +{ yygotominor.yy230 = (yymsp[-1].minor.yy230 & ~yymsp[0].minor.yy13.mask) | yymsp[0].minor.yy13.value; } break; case 74: /* refarg ::= MATCH nm */ case 75: /* refarg ::= ON INSERT refact */ yytestcase(yyruleno==75); -{ yygotominor.yy47.value = 0; yygotominor.yy47.mask = 0x000000; } +{ yygotominor.yy13.value = 0; yygotominor.yy13.mask = 0x000000; } break; case 76: /* refarg ::= ON DELETE refact */ -{ yygotominor.yy47.value = yymsp[0].minor.yy412; yygotominor.yy47.mask = 0x0000ff; } +{ yygotominor.yy13.value = yymsp[0].minor.yy230; yygotominor.yy13.mask = 0x0000ff; } break; case 77: /* refarg ::= ON UPDATE refact */ -{ yygotominor.yy47.value = yymsp[0].minor.yy412<<8; yygotominor.yy47.mask = 0x00ff00; } +{ yygotominor.yy13.value = yymsp[0].minor.yy230<<8; yygotominor.yy13.mask = 0x00ff00; } break; case 78: /* refact ::= SET NULL */ -{ yygotominor.yy412 = OE_SetNull; /* EV: R-33326-45252 */} +{ yygotominor.yy230 = OE_SetNull; /* EV: R-33326-45252 */} break; case 79: /* refact ::= SET DEFAULT */ -{ yygotominor.yy412 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yygotominor.yy230 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 80: /* refact ::= CASCADE */ -{ yygotominor.yy412 = OE_Cascade; /* EV: R-33326-45252 */} +{ yygotominor.yy230 = OE_Cascade; /* EV: R-33326-45252 */} break; case 81: /* refact ::= RESTRICT */ -{ yygotominor.yy412 = OE_Restrict; /* EV: R-33326-45252 */} +{ yygotominor.yy230 = OE_Restrict; /* EV: R-33326-45252 */} break; case 82: /* refact ::= NO ACTION */ -{ yygotominor.yy412 = OE_None; /* EV: R-33326-45252 */} +{ yygotominor.yy230 = OE_None; /* EV: R-33326-45252 */} break; case 84: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 99: /* defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==99); case 101: /* onconf ::= ON CONFLICT resolvetype */ yytestcase(yyruleno==101); case 104: /* resolvetype ::= raisetype */ yytestcase(yyruleno==104); -{yygotominor.yy412 = yymsp[0].minor.yy412;} +{yygotominor.yy230 = yymsp[0].minor.yy230;} break; case 88: /* conslist_opt ::= */ {yygotominor.yy0.n = 0; yygotominor.yy0.z = 0;} break; case 89: /* conslist_opt ::= COMMA conslist */ {yygotominor.yy0 = yymsp[-1].minor.yy0;} break; case 94: /* tcons ::= PRIMARY KEY LP idxlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy82,yymsp[0].minor.yy412,yymsp[-2].minor.yy412,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy462,yymsp[0].minor.yy230,yymsp[-2].minor.yy230,0);} break; case 95: /* tcons ::= UNIQUE LP idxlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy82,yymsp[0].minor.yy412,0,0,0,0);} +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy462,yymsp[0].minor.yy230,0,0,0,0);} break; case 96: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy22.pExpr);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy64.pExpr);} break; case 97: /* tcons ::= FOREIGN KEY LP idxlist RP REFERENCES nm idxlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy82, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy82, yymsp[-1].minor.yy412); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy412); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy462, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy462, yymsp[-1].minor.yy230); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy230); } break; case 100: /* onconf ::= */ -{yygotominor.yy412 = OE_Default;} +{yygotominor.yy230 = OE_Default;} break; case 102: /* orconf ::= */ -{yygotominor.yy18 = OE_Default;} +{yygotominor.yy44 = OE_Default;} break; case 103: /* orconf ::= OR resolvetype */ -{yygotominor.yy18 = (u8)yymsp[0].minor.yy412;} +{yygotominor.yy44 = (u8)yymsp[0].minor.yy230;} break; case 105: /* resolvetype ::= IGNORE */ -{yygotominor.yy412 = OE_Ignore;} +{yygotominor.yy230 = OE_Ignore;} break; case 106: /* resolvetype ::= REPLACE */ -{yygotominor.yy412 = OE_Replace;} +{yygotominor.yy230 = OE_Replace;} break; case 107: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy67, 0, yymsp[-1].minor.yy412); + sqlite3DropTable(pParse, yymsp[0].minor.yy285, 0, yymsp[-1].minor.yy230); } break; case 110: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm AS select */ { - sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy459, yymsp[-6].minor.yy412, yymsp[-4].minor.yy412); + sqlite3CreateView(pParse, &yymsp[-7].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, yymsp[0].minor.yy239, yymsp[-6].minor.yy230, yymsp[-4].minor.yy230); } break; case 111: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy67, 1, yymsp[-1].minor.yy412); + sqlite3DropTable(pParse, yymsp[0].minor.yy285, 1, yymsp[-1].minor.yy230); } break; case 112: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy459, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy459); + sqlite3Select(pParse, yymsp[0].minor.yy239, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy239); } break; case 113: /* select ::= oneselect */ -{yygotominor.yy459 = yymsp[0].minor.yy459;} +{yygotominor.yy239 = yymsp[0].minor.yy239;} break; case 114: /* select ::= select multiselect_op oneselect */ { - if( yymsp[0].minor.yy459 ){ - yymsp[0].minor.yy459->op = (u8)yymsp[-1].minor.yy412; - yymsp[0].minor.yy459->pPrior = yymsp[-2].minor.yy459; + if( yymsp[0].minor.yy239 ){ + yymsp[0].minor.yy239->op = (u8)yymsp[-1].minor.yy230; + yymsp[0].minor.yy239->pPrior = yymsp[-2].minor.yy239; }else{ - sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy459); + sqlite3SelectDelete(pParse->db, yymsp[-2].minor.yy239); } - yygotominor.yy459 = yymsp[0].minor.yy459; + yygotominor.yy239 = yymsp[0].minor.yy239; } break; case 116: /* multiselect_op ::= UNION ALL */ -{yygotominor.yy412 = TK_ALL;} +{yygotominor.yy230 = TK_ALL;} break; case 118: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { - yygotominor.yy459 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy82,yymsp[-5].minor.yy67,yymsp[-4].minor.yy2,yymsp[-3].minor.yy82,yymsp[-2].minor.yy2,yymsp[-1].minor.yy82,yymsp[-7].minor.yy412,yymsp[0].minor.yy244.pLimit,yymsp[0].minor.yy244.pOffset); + yygotominor.yy239 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy462,yymsp[-5].minor.yy285,yymsp[-4].minor.yy178,yymsp[-3].minor.yy462,yymsp[-2].minor.yy178,yymsp[-1].minor.yy462,yymsp[-7].minor.yy230,yymsp[0].minor.yy270.pLimit,yymsp[0].minor.yy270.pOffset); } break; case 122: /* sclp ::= selcollist COMMA */ - case 248: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==248); -{yygotominor.yy82 = yymsp[-1].minor.yy82;} + case 247: /* idxlist_opt ::= LP idxlist RP */ yytestcase(yyruleno==247); +{yygotominor.yy462 = yymsp[-1].minor.yy462;} break; case 123: /* sclp ::= */ case 151: /* orderby_opt ::= */ yytestcase(yyruleno==151); case 159: /* groupby_opt ::= */ yytestcase(yyruleno==159); - case 241: /* exprlist ::= */ yytestcase(yyruleno==241); - case 247: /* idxlist_opt ::= */ yytestcase(yyruleno==247); -{yygotominor.yy82 = 0;} + case 240: /* exprlist ::= */ yytestcase(yyruleno==240); + case 246: /* idxlist_opt ::= */ yytestcase(yyruleno==246); +{yygotominor.yy462 = 0;} break; case 124: /* selcollist ::= sclp expr as */ { - yygotominor.yy82 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy82, yymsp[-1].minor.yy22.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy82, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yygotominor.yy82,&yymsp[-1].minor.yy22); + yygotominor.yy462 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy462, yymsp[-1].minor.yy64.pExpr); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yygotominor.yy462, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yygotominor.yy462,&yymsp[-1].minor.yy64); } break; case 125: /* selcollist ::= sclp STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ALL, 0); - yygotominor.yy82 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy82, p); + yygotominor.yy462 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy462, p); } break; case 126: /* selcollist ::= sclp nm DOT STAR */ { Expr *pRight = sqlite3PExpr(pParse, TK_ALL, 0, 0, &yymsp[0].minor.yy0); Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy82, pDot); + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy462, pDot); } break; case 129: /* as ::= */ {yygotominor.yy0.n = 0;} break; case 130: /* from ::= */ -{yygotominor.yy67 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy67));} +{yygotominor.yy285 = sqlite3DbMallocZero(pParse->db, sizeof(*yygotominor.yy285));} break; case 131: /* from ::= FROM seltablist */ { - yygotominor.yy67 = yymsp[0].minor.yy67; - sqlite3SrcListShiftJoinType(yygotominor.yy67); + yygotominor.yy285 = yymsp[0].minor.yy285; + sqlite3SrcListShiftJoinType(yygotominor.yy285); } break; case 132: /* stl_prefix ::= seltablist joinop */ { - yygotominor.yy67 = yymsp[-1].minor.yy67; - if( ALWAYS(yygotominor.yy67 && yygotominor.yy67->nSrc>0) ) yygotominor.yy67->a[yygotominor.yy67->nSrc-1].jointype = (u8)yymsp[0].minor.yy412; + yygotominor.yy285 = yymsp[-1].minor.yy285; + if( ALWAYS(yygotominor.yy285 && yygotominor.yy285->nSrc>0) ) yygotominor.yy285->a[yygotominor.yy285->nSrc-1].jointype = (u8)yymsp[0].minor.yy230; } break; case 133: /* stl_prefix ::= */ -{yygotominor.yy67 = 0;} +{yygotominor.yy285 = 0;} break; case 134: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ { - yygotominor.yy67 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy67,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy2,yymsp[0].minor.yy240); - sqlite3SrcListIndexedBy(pParse, yygotominor.yy67, &yymsp[-2].minor.yy0); + yygotominor.yy285 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy285,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy178,yymsp[0].minor.yy160); + sqlite3SrcListIndexedBy(pParse, yygotominor.yy285, &yymsp[-2].minor.yy0); } break; case 135: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { - yygotominor.yy67 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy67,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy459,yymsp[-1].minor.yy2,yymsp[0].minor.yy240); + yygotominor.yy285 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy285,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy239,yymsp[-1].minor.yy178,yymsp[0].minor.yy160); } break; case 136: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy67==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy2==0 && yymsp[0].minor.yy240==0 ){ - yygotominor.yy67 = yymsp[-4].minor.yy67; + if( yymsp[-6].minor.yy285==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy178==0 && yymsp[0].minor.yy160==0 ){ + yygotominor.yy285 = yymsp[-4].minor.yy285; }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy67); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy67,0,0,0,0,0,0,0); - yygotominor.yy67 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy67,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy2,yymsp[0].minor.yy240); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy285); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy285,0,0,0,0,0,0,0); + yygotominor.yy285 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy285,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy178,yymsp[0].minor.yy160); } } break; case 137: /* dbnm ::= */ case 146: /* indexed_opt ::= */ yytestcase(yyruleno==146); {yygotominor.yy0.z=0; yygotominor.yy0.n=0;} break; case 139: /* fullname ::= nm dbnm */ -{yygotominor.yy67 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} +{yygotominor.yy285 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} break; case 140: /* joinop ::= COMMA|JOIN */ -{ yygotominor.yy412 = JT_INNER; } +{ yygotominor.yy230 = JT_INNER; } break; case 141: /* joinop ::= JOIN_KW JOIN */ -{ yygotominor.yy412 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } +{ yygotominor.yy230 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); } break; case 142: /* joinop ::= JOIN_KW nm JOIN */ -{ yygotominor.yy412 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } +{ yygotominor.yy230 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); } break; case 143: /* joinop ::= JOIN_KW nm nm JOIN */ -{ yygotominor.yy412 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } +{ yygotominor.yy230 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); } break; case 144: /* on_opt ::= ON expr */ case 155: /* sortitem ::= expr */ yytestcase(yyruleno==155); case 162: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==162); case 169: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==169); - case 236: /* case_else ::= ELSE expr */ yytestcase(yyruleno==236); - case 238: /* case_operand ::= expr */ yytestcase(yyruleno==238); -{yygotominor.yy2 = yymsp[0].minor.yy22.pExpr;} + case 235: /* case_else ::= ELSE expr */ yytestcase(yyruleno==235); + case 237: /* case_operand ::= expr */ yytestcase(yyruleno==237); +{yygotominor.yy178 = yymsp[0].minor.yy64.pExpr;} break; case 145: /* on_opt ::= */ case 161: /* having_opt ::= */ yytestcase(yyruleno==161); case 168: /* where_opt ::= */ yytestcase(yyruleno==168); - case 237: /* case_else ::= */ yytestcase(yyruleno==237); - case 239: /* case_operand ::= */ yytestcase(yyruleno==239); -{yygotominor.yy2 = 0;} + case 236: /* case_else ::= */ yytestcase(yyruleno==236); + case 238: /* case_operand ::= */ yytestcase(yyruleno==238); +{yygotominor.yy178 = 0;} break; case 148: /* indexed_opt ::= NOT INDEXED */ {yygotominor.yy0.z=0; yygotominor.yy0.n=1;} break; case 149: /* using_opt ::= USING LP inscollist RP */ case 181: /* inscollist_opt ::= LP inscollist RP */ yytestcase(yyruleno==181); -{yygotominor.yy240 = yymsp[-1].minor.yy240;} +{yygotominor.yy160 = yymsp[-1].minor.yy160;} break; case 150: /* using_opt ::= */ case 180: /* inscollist_opt ::= */ yytestcase(yyruleno==180); -{yygotominor.yy240 = 0;} +{yygotominor.yy160 = 0;} break; case 152: /* orderby_opt ::= ORDER BY sortlist */ case 160: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==160); - case 240: /* exprlist ::= nexprlist */ yytestcase(yyruleno==240); -{yygotominor.yy82 = yymsp[0].minor.yy82;} + case 239: /* exprlist ::= nexprlist */ yytestcase(yyruleno==239); +{yygotominor.yy462 = yymsp[0].minor.yy462;} break; case 153: /* sortlist ::= sortlist COMMA sortitem sortorder */ { - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy82,yymsp[-1].minor.yy2); - if( yygotominor.yy82 ) yygotominor.yy82->a[yygotominor.yy82->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy412; + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy462,yymsp[-1].minor.yy178); + if( yygotominor.yy462 ) yygotominor.yy462->a[yygotominor.yy462->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy230; } break; case 154: /* sortlist ::= sortitem sortorder */ { - yygotominor.yy82 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy2); - if( yygotominor.yy82 && ALWAYS(yygotominor.yy82->a) ) yygotominor.yy82->a[0].sortOrder = (u8)yymsp[0].minor.yy412; + yygotominor.yy462 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy178); + if( yygotominor.yy462 && ALWAYS(yygotominor.yy462->a) ) yygotominor.yy462->a[0].sortOrder = (u8)yymsp[0].minor.yy230; } break; case 156: /* sortorder ::= ASC */ case 158: /* sortorder ::= */ yytestcase(yyruleno==158); -{yygotominor.yy412 = SQLITE_SO_ASC;} +{yygotominor.yy230 = SQLITE_SO_ASC;} break; case 157: /* sortorder ::= DESC */ -{yygotominor.yy412 = SQLITE_SO_DESC;} +{yygotominor.yy230 = SQLITE_SO_DESC;} break; case 163: /* limit_opt ::= */ -{yygotominor.yy244.pLimit = 0; yygotominor.yy244.pOffset = 0;} +{yygotominor.yy270.pLimit = 0; yygotominor.yy270.pOffset = 0;} break; case 164: /* limit_opt ::= LIMIT expr */ -{yygotominor.yy244.pLimit = yymsp[0].minor.yy22.pExpr; yygotominor.yy244.pOffset = 0;} +{yygotominor.yy270.pLimit = yymsp[0].minor.yy64.pExpr; yygotominor.yy270.pOffset = 0;} break; case 165: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yygotominor.yy244.pLimit = yymsp[-2].minor.yy22.pExpr; yygotominor.yy244.pOffset = yymsp[0].minor.yy22.pExpr;} +{yygotominor.yy270.pLimit = yymsp[-2].minor.yy64.pExpr; yygotominor.yy270.pOffset = yymsp[0].minor.yy64.pExpr;} break; case 166: /* limit_opt ::= LIMIT expr COMMA expr */ -{yygotominor.yy244.pOffset = yymsp[-2].minor.yy22.pExpr; yygotominor.yy244.pLimit = yymsp[0].minor.yy22.pExpr;} +{yygotominor.yy270.pOffset = yymsp[-2].minor.yy64.pExpr; yygotominor.yy270.pLimit = yymsp[0].minor.yy64.pExpr;} break; case 167: /* cmd ::= DELETE FROM fullname indexed_opt where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy67, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy67,yymsp[0].minor.yy2); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy285, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy285,yymsp[0].minor.yy178); } break; case 170: /* cmd ::= UPDATE orconf fullname indexed_opt SET setlist where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy67, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy82,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy67,yymsp[-1].minor.yy82,yymsp[0].minor.yy2,yymsp[-5].minor.yy18); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy285, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy462,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy285,yymsp[-1].minor.yy462,yymsp[0].minor.yy178,yymsp[-5].minor.yy44); } break; case 171: /* setlist ::= setlist COMMA nm EQ expr */ { - yygotominor.yy82 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy82, yymsp[0].minor.yy22.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy82, &yymsp[-2].minor.yy0, 1); + yygotominor.yy462 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy462, yymsp[0].minor.yy64.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy462, &yymsp[-2].minor.yy0, 1); } break; case 172: /* setlist ::= nm EQ expr */ { - yygotominor.yy82 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy22.pExpr); - sqlite3ExprListSetName(pParse, yygotominor.yy82, &yymsp[-2].minor.yy0, 1); + yygotominor.yy462 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy64.pExpr); + sqlite3ExprListSetName(pParse, yygotominor.yy462, &yymsp[-2].minor.yy0, 1); } break; case 173: /* cmd ::= insert_cmd INTO fullname inscollist_opt VALUES LP itemlist RP */ -{sqlite3Insert(pParse, yymsp[-5].minor.yy67, yymsp[-1].minor.yy82, 0, yymsp[-4].minor.yy240, yymsp[-7].minor.yy18);} +{sqlite3Insert(pParse, yymsp[-5].minor.yy285, yymsp[-1].minor.yy462, 0, yymsp[-4].minor.yy160, yymsp[-7].minor.yy44);} break; case 174: /* cmd ::= insert_cmd INTO fullname inscollist_opt select */ -{sqlite3Insert(pParse, yymsp[-2].minor.yy67, 0, yymsp[0].minor.yy459, yymsp[-1].minor.yy240, yymsp[-4].minor.yy18);} +{sqlite3Insert(pParse, yymsp[-2].minor.yy285, 0, yymsp[0].minor.yy239, yymsp[-1].minor.yy160, yymsp[-4].minor.yy44);} break; case 175: /* cmd ::= insert_cmd INTO fullname inscollist_opt DEFAULT VALUES */ -{sqlite3Insert(pParse, yymsp[-3].minor.yy67, 0, 0, yymsp[-2].minor.yy240, yymsp[-5].minor.yy18);} +{sqlite3Insert(pParse, yymsp[-3].minor.yy285, 0, 0, yymsp[-2].minor.yy160, yymsp[-5].minor.yy44);} break; case 176: /* insert_cmd ::= INSERT orconf */ -{yygotominor.yy18 = yymsp[0].minor.yy18;} +{yygotominor.yy44 = yymsp[0].minor.yy44;} break; case 177: /* insert_cmd ::= REPLACE */ -{yygotominor.yy18 = OE_Replace;} +{yygotominor.yy44 = OE_Replace;} break; case 178: /* itemlist ::= itemlist COMMA expr */ - case 242: /* nexprlist ::= nexprlist COMMA expr */ yytestcase(yyruleno==242); -{yygotominor.yy82 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy82,yymsp[0].minor.yy22.pExpr);} + case 241: /* nexprlist ::= nexprlist COMMA expr */ yytestcase(yyruleno==241); +{yygotominor.yy462 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy462,yymsp[0].minor.yy64.pExpr);} break; case 179: /* itemlist ::= expr */ - case 243: /* nexprlist ::= expr */ yytestcase(yyruleno==243); -{yygotominor.yy82 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy22.pExpr);} + case 242: /* nexprlist ::= expr */ yytestcase(yyruleno==242); +{yygotominor.yy462 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy64.pExpr);} break; case 182: /* inscollist ::= inscollist COMMA nm */ -{yygotominor.yy240 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy240,&yymsp[0].minor.yy0);} +{yygotominor.yy160 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy160,&yymsp[0].minor.yy0);} break; case 183: /* inscollist ::= nm */ -{yygotominor.yy240 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} +{yygotominor.yy160 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0);} break; case 184: /* expr ::= term */ - case 212: /* escape ::= ESCAPE expr */ yytestcase(yyruleno==212); -{yygotominor.yy22 = yymsp[0].minor.yy22;} +{yygotominor.yy64 = yymsp[0].minor.yy64;} break; case 185: /* expr ::= LP expr RP */ -{yygotominor.yy22.pExpr = yymsp[-1].minor.yy22.pExpr; spanSet(&yygotominor.yy22,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} +{yygotominor.yy64.pExpr = yymsp[-1].minor.yy64.pExpr; spanSet(&yygotominor.yy64,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0);} break; case 186: /* term ::= NULL */ case 191: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==191); case 192: /* term ::= STRING */ yytestcase(yyruleno==192); -{spanExpr(&yygotominor.yy22, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} +{spanExpr(&yygotominor.yy64, pParse, yymsp[0].major, &yymsp[0].minor.yy0);} break; case 187: /* expr ::= id */ case 188: /* expr ::= JOIN_KW */ yytestcase(yyruleno==188); -{spanExpr(&yygotominor.yy22, pParse, TK_ID, &yymsp[0].minor.yy0);} +{spanExpr(&yygotominor.yy64, pParse, TK_ID, &yymsp[0].minor.yy0);} break; case 189: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); - spanSet(&yygotominor.yy22,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + spanSet(&yygotominor.yy64,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); } break; case 190: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); - spanSet(&yygotominor.yy22,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + spanSet(&yygotominor.yy64,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); } break; case 193: /* expr ::= REGISTER */ { /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &yymsp[0].minor.yy0); - yygotominor.yy22.pExpr = 0; + yygotominor.yy64.pExpr = 0; }else{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); - if( yygotominor.yy22.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy22.pExpr->iTable); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &yymsp[0].minor.yy0); + if( yygotominor.yy64.pExpr ) sqlite3GetInt32(&yymsp[0].minor.yy0.z[1], &yygotominor.yy64.pExpr->iTable); } - spanSet(&yygotominor.yy22, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy64, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; case 194: /* expr ::= VARIABLE */ { - spanExpr(&yygotominor.yy22, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yygotominor.yy22.pExpr); - spanSet(&yygotominor.yy22, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + spanExpr(&yygotominor.yy64, pParse, TK_VARIABLE, &yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yygotominor.yy64.pExpr); + spanSet(&yygotominor.yy64, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; case 195: /* expr ::= expr COLLATE ids */ { - yygotominor.yy22.pExpr = sqlite3ExprSetColl(pParse, yymsp[-2].minor.yy22.pExpr, &yymsp[0].minor.yy0); - yygotominor.yy22.zStart = yymsp[-2].minor.yy22.zStart; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yygotominor.yy64.pExpr = sqlite3ExprSetCollByToken(pParse, yymsp[-2].minor.yy64.pExpr, &yymsp[0].minor.yy0); + yygotominor.yy64.zStart = yymsp[-2].minor.yy64.zStart; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; case 196: /* expr ::= CAST LP expr AS typetoken RP */ { - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy22.pExpr, 0, &yymsp[-1].minor.yy0); - spanSet(&yygotominor.yy22,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy64.pExpr, 0, &yymsp[-1].minor.yy0); + spanSet(&yygotominor.yy64,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); } break; case 197: /* expr ::= ID LP distinct exprlist RP */ { - if( yymsp[-1].minor.yy82 && yymsp[-1].minor.yy82->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + if( yymsp[-1].minor.yy462 && yymsp[-1].minor.yy462->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); } - yygotominor.yy22.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy82, &yymsp[-4].minor.yy0); - spanSet(&yygotominor.yy22,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy412 && yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->flags |= EP_Distinct; + yygotominor.yy64.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy462, &yymsp[-4].minor.yy0); + spanSet(&yygotominor.yy64,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); + if( yymsp[-2].minor.yy230 && yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->flags |= EP_Distinct; } } break; case 198: /* expr ::= ID LP STAR RP */ { - yygotominor.yy22.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yygotominor.yy22,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + yygotominor.yy64.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + spanSet(&yygotominor.yy64,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); } break; case 199: /* term ::= CTIME_KW */ { /* The CURRENT_TIME, CURRENT_DATE, and CURRENT_TIMESTAMP values are ** treated as functions that return constants */ - yygotominor.yy22.pExpr = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->op = TK_CONST_FUNC; + yygotominor.yy64.pExpr = sqlite3ExprFunction(pParse, 0,&yymsp[0].minor.yy0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->op = TK_CONST_FUNC; } - spanSet(&yygotominor.yy22, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + spanSet(&yygotominor.yy64, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); } break; case 200: /* expr ::= expr AND expr */ case 201: /* expr ::= expr OR expr */ yytestcase(yyruleno==201); case 202: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==202); @@ -100111,343 +102602,363 @@ case 203: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==203); case 204: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==204); case 205: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==205); case 206: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==206); case 207: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==207); -{spanBinaryExpr(&yygotominor.yy22,pParse,yymsp[-1].major,&yymsp[-2].minor.yy22,&yymsp[0].minor.yy22);} +{spanBinaryExpr(&yygotominor.yy64,pParse,yymsp[-1].major,&yymsp[-2].minor.yy64,&yymsp[0].minor.yy64);} break; case 208: /* likeop ::= LIKE_KW */ case 210: /* likeop ::= MATCH */ yytestcase(yyruleno==210); -{yygotominor.yy438.eOperator = yymsp[0].minor.yy0; yygotominor.yy438.not = 0;} +{yygotominor.yy440.eOperator = yymsp[0].minor.yy0; yygotominor.yy440.not = 0;} break; case 209: /* likeop ::= NOT LIKE_KW */ case 211: /* likeop ::= NOT MATCH */ yytestcase(yyruleno==211); -{yygotominor.yy438.eOperator = yymsp[0].minor.yy0; yygotominor.yy438.not = 1;} +{yygotominor.yy440.eOperator = yymsp[0].minor.yy0; yygotominor.yy440.not = 1;} break; - case 213: /* escape ::= */ -{memset(&yygotominor.yy22,0,sizeof(yygotominor.yy22));} + case 212: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy64.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy64.pExpr); + yygotominor.yy64.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy440.eOperator); + if( yymsp[-1].minor.yy440.not ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + yygotominor.yy64.zStart = yymsp[-2].minor.yy64.zStart; + yygotominor.yy64.zEnd = yymsp[0].minor.yy64.zEnd; + if( yygotominor.yy64.pExpr ) yygotominor.yy64.pExpr->flags |= EP_InfixFunc; +} break; - case 214: /* expr ::= expr likeop expr escape */ + case 213: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-1].minor.yy22.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-3].minor.yy22.pExpr); - if( yymsp[0].minor.yy22.pExpr ){ - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy22.pExpr); - } - yygotominor.yy22.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-2].minor.yy438.eOperator); - if( yymsp[-2].minor.yy438.not ) yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy22.pExpr, 0, 0); - yygotominor.yy22.zStart = yymsp[-3].minor.yy22.zStart; - yygotominor.yy22.zEnd = yymsp[-1].minor.yy22.zEnd; - if( yygotominor.yy22.pExpr ) yygotominor.yy22.pExpr->flags |= EP_InfixFunc; -} - break; - case 215: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(&yygotominor.yy22,pParse,yymsp[0].major,&yymsp[-1].minor.yy22,&yymsp[0].minor.yy0);} - break; - case 216: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(&yygotominor.yy22,pParse,TK_NOTNULL,&yymsp[-2].minor.yy22,&yymsp[0].minor.yy0);} - break; - case 217: /* expr ::= expr IS expr */ -{ - spanBinaryExpr(&yygotominor.yy22,pParse,TK_IS,&yymsp[-2].minor.yy22,&yymsp[0].minor.yy22); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy22.pExpr, yygotominor.yy22.pExpr, TK_ISNULL); -} - break; - case 218: /* expr ::= expr IS NOT expr */ -{ - spanBinaryExpr(&yygotominor.yy22,pParse,TK_ISNOT,&yymsp[-3].minor.yy22,&yymsp[0].minor.yy22); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy22.pExpr, yygotominor.yy22.pExpr, TK_NOTNULL); -} - break; - case 219: /* expr ::= NOT expr */ - case 220: /* expr ::= BITNOT expr */ yytestcase(yyruleno==220); -{spanUnaryPrefix(&yygotominor.yy22,pParse,yymsp[-1].major,&yymsp[0].minor.yy22,&yymsp[-1].minor.yy0);} - break; - case 221: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yygotominor.yy22,pParse,TK_UMINUS,&yymsp[0].minor.yy22,&yymsp[-1].minor.yy0);} - break; - case 222: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yygotominor.yy22,pParse,TK_UPLUS,&yymsp[0].minor.yy22,&yymsp[-1].minor.yy0);} - break; - case 225: /* expr ::= expr between_op expr AND expr */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy22.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy22.pExpr); - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy22.pExpr, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pList = pList; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy64.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy64.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy64.pExpr); + yygotominor.yy64.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy440.eOperator); + if( yymsp[-3].minor.yy440.not ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + yygotominor.yy64.zStart = yymsp[-4].minor.yy64.zStart; + yygotominor.yy64.zEnd = yymsp[0].minor.yy64.zEnd; + if( yygotominor.yy64.pExpr ) yygotominor.yy64.pExpr->flags |= EP_InfixFunc; +} + break; + case 214: /* expr ::= expr ISNULL|NOTNULL */ +{spanUnaryPostfix(&yygotominor.yy64,pParse,yymsp[0].major,&yymsp[-1].minor.yy64,&yymsp[0].minor.yy0);} + break; + case 215: /* expr ::= expr NOT NULL */ +{spanUnaryPostfix(&yygotominor.yy64,pParse,TK_NOTNULL,&yymsp[-2].minor.yy64,&yymsp[0].minor.yy0);} + break; + case 216: /* expr ::= expr IS expr */ +{ + spanBinaryExpr(&yygotominor.yy64,pParse,TK_IS,&yymsp[-2].minor.yy64,&yymsp[0].minor.yy64); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy64.pExpr, yygotominor.yy64.pExpr, TK_ISNULL); +} + break; + case 217: /* expr ::= expr IS NOT expr */ +{ + spanBinaryExpr(&yygotominor.yy64,pParse,TK_ISNOT,&yymsp[-3].minor.yy64,&yymsp[0].minor.yy64); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy64.pExpr, yygotominor.yy64.pExpr, TK_NOTNULL); +} + break; + case 218: /* expr ::= NOT expr */ + case 219: /* expr ::= BITNOT expr */ yytestcase(yyruleno==219); +{spanUnaryPrefix(&yygotominor.yy64,pParse,yymsp[-1].major,&yymsp[0].minor.yy64,&yymsp[-1].minor.yy0);} + break; + case 220: /* expr ::= MINUS expr */ +{spanUnaryPrefix(&yygotominor.yy64,pParse,TK_UMINUS,&yymsp[0].minor.yy64,&yymsp[-1].minor.yy0);} + break; + case 221: /* expr ::= PLUS expr */ +{spanUnaryPrefix(&yygotominor.yy64,pParse,TK_UPLUS,&yymsp[0].minor.yy64,&yymsp[-1].minor.yy0);} + break; + case 224: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy64.pExpr); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy64.pExpr); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy64.pExpr, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy412 ) yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy22.pExpr, 0, 0); - yygotominor.yy22.zStart = yymsp[-4].minor.yy22.zStart; - yygotominor.yy22.zEnd = yymsp[0].minor.yy22.zEnd; -} - break; - case 228: /* expr ::= expr in_op LP exprlist RP */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy22.pExpr, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pList = yymsp[-1].minor.yy82; - sqlite3ExprSetHeight(pParse, yygotominor.yy22.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy82); - } - if( yymsp[-3].minor.yy412 ) yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy22.pExpr, 0, 0); - yygotominor.yy22.zStart = yymsp[-4].minor.yy22.zStart; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 229: /* expr ::= LP select RP */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pSelect = yymsp[-1].minor.yy459; - ExprSetProperty(yygotominor.yy22.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy22.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy459); - } - yygotominor.yy22.zStart = yymsp[-2].minor.yy0.z; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 230: /* expr ::= expr in_op LP select RP */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy22.pExpr, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pSelect = yymsp[-1].minor.yy459; - ExprSetProperty(yygotominor.yy22.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy22.pExpr); - }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy459); - } - if( yymsp[-3].minor.yy412 ) yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy22.pExpr, 0, 0); - yygotominor.yy22.zStart = yymsp[-4].minor.yy22.zStart; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 231: /* expr ::= expr in_op nm dbnm */ + if( yymsp[-3].minor.yy230 ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + yygotominor.yy64.zStart = yymsp[-4].minor.yy64.zStart; + yygotominor.yy64.zEnd = yymsp[0].minor.yy64.zEnd; +} + break; + case 227: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy462==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy230]); + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy64.pExpr); + }else{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy64.pExpr, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pList = yymsp[-1].minor.yy462; + sqlite3ExprSetHeight(pParse, yygotominor.yy64.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy462); + } + if( yymsp[-3].minor.yy230 ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + } + yygotominor.yy64.zStart = yymsp[-4].minor.yy64.zStart; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 228: /* expr ::= LP select RP */ +{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pSelect = yymsp[-1].minor.yy239; + ExprSetProperty(yygotominor.yy64.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy64.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy239); + } + yygotominor.yy64.zStart = yymsp[-2].minor.yy0.z; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 229: /* expr ::= expr in_op LP select RP */ +{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy64.pExpr, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pSelect = yymsp[-1].minor.yy239; + ExprSetProperty(yygotominor.yy64.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy64.pExpr); + }else{ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy239); + } + if( yymsp[-3].minor.yy230 ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + yygotominor.yy64.zStart = yymsp[-4].minor.yy64.zStart; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 230: /* expr ::= expr in_op nm dbnm */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy22.pExpr, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(yygotominor.yy22.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, yygotominor.yy22.pExpr); + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy64.pExpr, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); + ExprSetProperty(yygotominor.yy64.pExpr, EP_xIsSelect); + sqlite3ExprSetHeight(pParse, yygotominor.yy64.pExpr); }else{ sqlite3SrcListDelete(pParse->db, pSrc); } - if( yymsp[-2].minor.yy412 ) yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy22.pExpr, 0, 0); - yygotominor.yy22.zStart = yymsp[-3].minor.yy22.zStart; - yygotominor.yy22.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; - } - break; - case 232: /* expr ::= EXISTS LP select RP */ -{ - Expr *p = yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); + if( yymsp[-2].minor.yy230 ) yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_NOT, yygotominor.yy64.pExpr, 0, 0); + yygotominor.yy64.zStart = yymsp[-3].minor.yy64.zStart; + yygotominor.yy64.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + } + break; + case 231: /* expr ::= EXISTS LP select RP */ +{ + Expr *p = yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ - p->x.pSelect = yymsp[-1].minor.yy459; + p->x.pSelect = yymsp[-1].minor.yy239; ExprSetProperty(p, EP_xIsSelect); sqlite3ExprSetHeight(pParse, p); }else{ - sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy459); - } - yygotominor.yy22.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; - } - break; - case 233: /* expr ::= CASE case_operand case_exprlist case_else END */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy2, yymsp[-1].minor.yy2, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->x.pList = yymsp[-2].minor.yy82; - sqlite3ExprSetHeight(pParse, yygotominor.yy22.pExpr); - }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy82); - } - yygotominor.yy22.zStart = yymsp[-4].minor.yy0.z; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 234: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ -{ - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy82, yymsp[-2].minor.yy22.pExpr); - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yygotominor.yy82, yymsp[0].minor.yy22.pExpr); -} - break; - case 235: /* case_exprlist ::= WHEN expr THEN expr */ -{ - yygotominor.yy82 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy22.pExpr); - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yygotominor.yy82, yymsp[0].minor.yy22.pExpr); -} - break; - case 244: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ + sqlite3SelectDelete(pParse->db, yymsp[-1].minor.yy239); + } + yygotominor.yy64.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + } + break; + case 232: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy178, yymsp[-1].minor.yy178, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->x.pList = yymsp[-2].minor.yy462; + sqlite3ExprSetHeight(pParse, yygotominor.yy64.pExpr); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy462); + } + yygotominor.yy64.zStart = yymsp[-4].minor.yy0.z; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 233: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy462, yymsp[-2].minor.yy64.pExpr); + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yygotominor.yy462, yymsp[0].minor.yy64.pExpr); +} + break; + case 234: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yygotominor.yy462 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy64.pExpr); + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yygotominor.yy462, yymsp[0].minor.yy64.pExpr); +} + break; + case 243: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP idxlist RP */ { sqlite3CreateIndex(pParse, &yymsp[-6].minor.yy0, &yymsp[-5].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy82, yymsp[-9].minor.yy412, - &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy412); -} - break; - case 245: /* uniqueflag ::= UNIQUE */ - case 287: /* raisetype ::= ABORT */ yytestcase(yyruleno==287); -{yygotominor.yy412 = OE_Abort;} - break; - case 246: /* uniqueflag ::= */ -{yygotominor.yy412 = OE_None;} - break; - case 249: /* idxlist ::= idxlist COMMA nm collate sortorder */ + sqlite3SrcListAppend(pParse->db,0,&yymsp[-3].minor.yy0,0), yymsp[-1].minor.yy462, yymsp[-9].minor.yy230, + &yymsp[-10].minor.yy0, &yymsp[0].minor.yy0, SQLITE_SO_ASC, yymsp[-7].minor.yy230); +} + break; + case 244: /* uniqueflag ::= UNIQUE */ + case 286: /* raisetype ::= ABORT */ yytestcase(yyruleno==286); +{yygotominor.yy230 = OE_Abort;} + break; + case 245: /* uniqueflag ::= */ +{yygotominor.yy230 = OE_None;} + break; + case 248: /* idxlist ::= idxlist COMMA nm collate sortorder */ { Expr *p = 0; if( yymsp[-1].minor.yy0.n>0 ){ p = sqlite3Expr(pParse->db, TK_COLUMN, 0); - sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); } - yygotominor.yy82 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy82, p); - sqlite3ExprListSetName(pParse,yygotominor.yy82,&yymsp[-2].minor.yy0,1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy82, "index"); - if( yygotominor.yy82 ) yygotominor.yy82->a[yygotominor.yy82->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy412; + yygotominor.yy462 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy462, p); + sqlite3ExprListSetName(pParse,yygotominor.yy462,&yymsp[-2].minor.yy0,1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy462, "index"); + if( yygotominor.yy462 ) yygotominor.yy462->a[yygotominor.yy462->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy230; } break; - case 250: /* idxlist ::= nm collate sortorder */ + case 249: /* idxlist ::= nm collate sortorder */ { Expr *p = 0; if( yymsp[-1].minor.yy0.n>0 ){ p = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); - sqlite3ExprSetColl(pParse, p, &yymsp[-1].minor.yy0); + sqlite3ExprSetCollByToken(pParse, p, &yymsp[-1].minor.yy0); } - yygotominor.yy82 = sqlite3ExprListAppend(pParse,0, p); - sqlite3ExprListSetName(pParse, yygotominor.yy82, &yymsp[-2].minor.yy0, 1); - sqlite3ExprListCheckLength(pParse, yygotominor.yy82, "index"); - if( yygotominor.yy82 ) yygotominor.yy82->a[yygotominor.yy82->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy412; + yygotominor.yy462 = sqlite3ExprListAppend(pParse,0, p); + sqlite3ExprListSetName(pParse, yygotominor.yy462, &yymsp[-2].minor.yy0, 1); + sqlite3ExprListCheckLength(pParse, yygotominor.yy462, "index"); + if( yygotominor.yy462 ) yygotominor.yy462->a[yygotominor.yy462->nExpr-1].sortOrder = (u8)yymsp[0].minor.yy230; } break; - case 251: /* collate ::= */ + case 250: /* collate ::= */ {yygotominor.yy0.z = 0; yygotominor.yy0.n = 0;} break; - case 253: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy67, yymsp[-1].minor.yy412);} + case 252: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy285, yymsp[-1].minor.yy230);} break; - case 259: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy347, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy247, &all); } break; - case 260: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy412, yymsp[-4].minor.yy210.a, yymsp[-4].minor.yy210.b, yymsp[-2].minor.yy67, yymsp[0].minor.yy2, yymsp[-10].minor.yy412, yymsp[-8].minor.yy412); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy230, yymsp[-4].minor.yy132.a, yymsp[-4].minor.yy132.b, yymsp[-2].minor.yy285, yymsp[0].minor.yy178, yymsp[-10].minor.yy230, yymsp[-8].minor.yy230); yygotominor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); } break; - case 261: /* trigger_time ::= BEFORE */ - case 264: /* trigger_time ::= */ yytestcase(yyruleno==264); -{ yygotominor.yy412 = TK_BEFORE; } - break; - case 262: /* trigger_time ::= AFTER */ -{ yygotominor.yy412 = TK_AFTER; } - break; - case 263: /* trigger_time ::= INSTEAD OF */ -{ yygotominor.yy412 = TK_INSTEAD;} - break; - case 265: /* trigger_event ::= DELETE|INSERT */ - case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); -{yygotominor.yy210.a = yymsp[0].major; yygotominor.yy210.b = 0;} - break; - case 267: /* trigger_event ::= UPDATE OF inscollist */ -{yygotominor.yy210.a = TK_UPDATE; yygotominor.yy210.b = yymsp[0].minor.yy240;} - break; - case 270: /* when_clause ::= */ -{ yygotominor.yy2 = 0; } - break; - case 271: /* when_clause ::= WHEN expr */ -{ yygotominor.yy2 = yymsp[0].minor.yy22.pExpr; } - break; - case 272: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ -{ - assert( yymsp[-2].minor.yy347!=0 ); - yymsp[-2].minor.yy347->pLast->pNext = yymsp[-1].minor.yy347; - yymsp[-2].minor.yy347->pLast = yymsp[-1].minor.yy347; - yygotominor.yy347 = yymsp[-2].minor.yy347; -} - break; - case 273: /* trigger_cmd_list ::= trigger_cmd SEMI */ -{ - assert( yymsp[-1].minor.yy347!=0 ); - yymsp[-1].minor.yy347->pLast = yymsp[-1].minor.yy347; - yygotominor.yy347 = yymsp[-1].minor.yy347; -} - break; - case 275: /* trnm ::= nm DOT nm */ + case 260: /* trigger_time ::= BEFORE */ + case 263: /* trigger_time ::= */ yytestcase(yyruleno==263); +{ yygotominor.yy230 = TK_BEFORE; } + break; + case 261: /* trigger_time ::= AFTER */ +{ yygotominor.yy230 = TK_AFTER; } + break; + case 262: /* trigger_time ::= INSTEAD OF */ +{ yygotominor.yy230 = TK_INSTEAD;} + break; + case 264: /* trigger_event ::= DELETE|INSERT */ + case 265: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==265); +{yygotominor.yy132.a = yymsp[0].major; yygotominor.yy132.b = 0;} + break; + case 266: /* trigger_event ::= UPDATE OF inscollist */ +{yygotominor.yy132.a = TK_UPDATE; yygotominor.yy132.b = yymsp[0].minor.yy160;} + break; + case 269: /* when_clause ::= */ +{ yygotominor.yy178 = 0; } + break; + case 270: /* when_clause ::= WHEN expr */ +{ yygotominor.yy178 = yymsp[0].minor.yy64.pExpr; } + break; + case 271: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy247!=0 ); + yymsp[-2].minor.yy247->pLast->pNext = yymsp[-1].minor.yy247; + yymsp[-2].minor.yy247->pLast = yymsp[-1].minor.yy247; + yygotominor.yy247 = yymsp[-2].minor.yy247; +} + break; + case 272: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy247!=0 ); + yymsp[-1].minor.yy247->pLast = yymsp[-1].minor.yy247; + yygotominor.yy247 = yymsp[-1].minor.yy247; +} + break; + case 274: /* trnm ::= nm DOT nm */ { yygotominor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, "qualified table names are not allowed on INSERT, UPDATE, and DELETE " "statements within triggers"); } break; - case 277: /* tridxby ::= INDEXED BY nm */ + case 276: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 278: /* tridxby ::= NOT INDEXED */ + case 277: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 279: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{ yygotominor.yy347 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy82, yymsp[0].minor.yy2, yymsp[-5].minor.yy18); } - break; - case 280: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP */ -{yygotominor.yy347 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy240, yymsp[-1].minor.yy82, 0, yymsp[-7].minor.yy18);} - break; - case 281: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ -{yygotominor.yy347 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy240, 0, yymsp[0].minor.yy459, yymsp[-4].minor.yy18);} - break; - case 282: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yygotominor.yy347 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy2);} - break; - case 283: /* trigger_cmd ::= select */ -{yygotominor.yy347 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy459); } - break; - case 284: /* expr ::= RAISE LP IGNORE RP */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yygotominor.yy22.pExpr ){ - yygotominor.yy22.pExpr->affinity = OE_Ignore; - } - yygotominor.yy22.zStart = yymsp[-3].minor.yy0.z; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 285: /* expr ::= RAISE LP raisetype COMMA nm RP */ -{ - yygotominor.yy22.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yygotominor.yy22.pExpr ) { - yygotominor.yy22.pExpr->affinity = (char)yymsp[-3].minor.yy412; - } - yygotominor.yy22.zStart = yymsp[-5].minor.yy0.z; - yygotominor.yy22.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; -} - break; - case 286: /* raisetype ::= ROLLBACK */ -{yygotominor.yy412 = OE_Rollback;} - break; - case 288: /* raisetype ::= FAIL */ -{yygotominor.yy412 = OE_Fail;} - break; - case 289: /* cmd ::= DROP TRIGGER ifexists fullname */ -{ - sqlite3DropTrigger(pParse,yymsp[0].minor.yy67,yymsp[-1].minor.yy412); + case 278: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ +{ yygotominor.yy247 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy462, yymsp[0].minor.yy178, yymsp[-5].minor.yy44); } + break; + case 279: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt VALUES LP itemlist RP */ +{yygotominor.yy247 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy160, yymsp[-1].minor.yy462, 0, yymsp[-7].minor.yy44);} + break; + case 280: /* trigger_cmd ::= insert_cmd INTO trnm inscollist_opt select */ +{yygotominor.yy247 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy160, 0, yymsp[0].minor.yy239, yymsp[-4].minor.yy44);} + break; + case 281: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ +{yygotominor.yy247 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy178);} + break; + case 282: /* trigger_cmd ::= select */ +{yygotominor.yy247 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy239); } + break; + case 283: /* expr ::= RAISE LP IGNORE RP */ +{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); + if( yygotominor.yy64.pExpr ){ + yygotominor.yy64.pExpr->affinity = OE_Ignore; + } + yygotominor.yy64.zStart = yymsp[-3].minor.yy0.z; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 284: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yygotominor.yy64.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); + if( yygotominor.yy64.pExpr ) { + yygotominor.yy64.pExpr->affinity = (char)yymsp[-3].minor.yy230; + } + yygotominor.yy64.zStart = yymsp[-5].minor.yy0.z; + yygotominor.yy64.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; +} + break; + case 285: /* raisetype ::= ROLLBACK */ +{yygotominor.yy230 = OE_Rollback;} + break; + case 287: /* raisetype ::= FAIL */ +{yygotominor.yy230 = OE_Fail;} + break; + case 288: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy285,yymsp[-1].minor.yy230); } break; default: /* (0) input ::= cmdlist */ yytestcase(yyruleno==0); /* (1) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==1); @@ -100472,15 +102983,15 @@ /* (62) ccons ::= NULL onconf */ yytestcase(yyruleno==62); /* (90) conslist ::= conslist COMMA tcons */ yytestcase(yyruleno==90); /* (91) conslist ::= conslist tcons */ yytestcase(yyruleno==91); /* (92) conslist ::= tcons */ yytestcase(yyruleno==92); /* (93) tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==93); - /* (257) plus_opt ::= PLUS */ yytestcase(yyruleno==257); - /* (258) plus_opt ::= */ yytestcase(yyruleno==258); - /* (268) foreach_clause ::= */ yytestcase(yyruleno==268); - /* (269) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==269); - /* (276) tridxby ::= */ yytestcase(yyruleno==276); + /* (256) plus_opt ::= PLUS */ yytestcase(yyruleno==256); + /* (257) plus_opt ::= */ yytestcase(yyruleno==257); + /* (267) foreach_clause ::= */ yytestcase(yyruleno==267); + /* (268) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==268); + /* (275) tridxby ::= */ yytestcase(yyruleno==275); break; }; yygoto = yyRuleInfo[yyruleno].lhs; yysize = yyRuleInfo[yyruleno].nrhs; yypParser->yyidx -= yysize; @@ -101477,19 +103988,19 @@ pParse->aTableLock = 0; pParse->nTableLock = 0; } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3DbFree(db, pParse->apVtabLock); + sqlite3_free(pParse->apVtabLock); #endif if( !IN_DECLARE_VTAB ){ /* If the pParse->declareVtab flag is set, do not delete any table ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ - sqlite3DeleteTable(pParse->pNewTable); + sqlite3DeleteTable(db, pParse->pNewTable); } sqlite3DeleteTrigger(db, pParse->pNewTrigger); sqlite3DbFree(db, pParse->apVarExpr); sqlite3DbFree(db, pParse->aAlias); @@ -101499,11 +104010,11 @@ sqlite3DbFree(db, p); } while( pParse->pZombieTab ){ Table *p = pParse->pZombieTab; pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(p); + sqlite3DeleteTable(db, p); } if( nErr>0 && pParse->rc==SQLITE_OK ){ pParse->rc = SQLITE_ERROR; } return nErr; @@ -101909,19 +104420,37 @@ /************** End of sqliteicu.h *******************************************/ /************** Continuing where we left off in main.c ***********************/ #endif -/* -** The version of the library -*/ #ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; #endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. +*/ SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-54823-41343 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } #if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* ** If the following function pointer is not NULL and if @@ -102038,10 +104567,17 @@ /* Do the rest of the initialization under the recursive mutex so ** that we will be able to handle recursive calls into ** sqlite3_initialize(). The recursive calls normally come through ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). */ sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); sqlite3GlobalConfig.inProgress = 1; @@ -102318,16 +104854,16 @@ if( cnt<0 ) cnt = 0; if( sz==0 || cnt==0 ){ sz = 0; pStart = 0; }else if( pBuf==0 ){ - sz = ROUND8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); + pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); }else{ - sz = ROUNDDOWN8(sz); + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ pStart = pBuf; } db->lookaside.pStart = pStart; db->lookaside.pFree = 0; db->lookaside.sz = (u16)sz; @@ -102366,18 +104902,18 @@ va_list ap; int rc; va_start(ap, op); switch( op ){ case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); - int sz = va_arg(ap, int); - int cnt = va_arg(ap, int); + void *pBuf = va_arg(ap, void*); /* IMP: R-21112-12275 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ rc = setupLookaside(db, pBuf, sz, cnt); break; } default: { - rc = SQLITE_ERROR; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ break; } } va_end(ap); return rc; @@ -102477,16 +105013,33 @@ } db->nSavepoint = 0; db->nStatement = 0; db->isTransactionSavepoint = 0; } + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} /* ** Close an existing SQLite database */ SQLITE_API int sqlite3_close(sqlite3 *db){ - HashElem *i; + HashElem *i; /* Hash table iterator */ int j; if( !db ){ return SQLITE_OK; } @@ -102550,10 +105103,11 @@ for(j=0; jaFunc.a); j++){ FuncDef *pNext, *pHash, *p; for(p=db->aFunc.a[j]; p; p=pHash){ pHash = p->pHash; while( p ){ + functionDestroy(db, p); pNext = p->pNext; sqlite3DbFree(db, p); p = pNext; } } @@ -102824,11 +105378,12 @@ int nArg, int enc, void *pUserData, void (*xFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) + void (*xFinal)(sqlite3_context*), + FuncDestructor *pDestructor ){ FuncDef *p; int nName; assert( sqlite3_mutex_held(db->mutex) ); @@ -102852,14 +105407,14 @@ if( enc==SQLITE_UTF16 ){ enc = SQLITE_UTF16NATIVE; }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE, - pUserData, xFunc, xStep, xFinal); + pUserData, xFunc, xStep, xFinal, pDestructor); } if( rc!=SQLITE_OK ){ return rc; } enc = SQLITE_UTF16BE; @@ -102888,10 +105443,19 @@ p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 1); assert(p || db->mallocFailed); if( !p ){ return SQLITE_NOMEM; } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->pDestructor = pDestructor; p->flags = 0; p->xFunc = xFunc; p->xStep = xStep; p->xFinalize = xFinal; p->pUserData = pUserData; @@ -102902,21 +105466,53 @@ /* ** Create new user functions. */ SQLITE_API int sqlite3_create_function( sqlite3 *db, - const char *zFunctionName, + const char *zFunc, int nArg, int enc, void *p, void (*xFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*) ){ - int rc; + return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xFunc, xStep, + xFinal, 0); +} + +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; sqlite3_mutex_enter(db->mutex); - rc = sqlite3CreateFunc(db, zFunctionName, nArg, enc, p, xFunc, xStep, xFinal); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + if( !pArg ){ + xDestroy(p); + goto out; + } + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xFunc, xStep, xFinal, pArg); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3DbFree(db, pArg); + } + + out: rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -102934,11 +105530,11 @@ int rc; char *zFunc8; sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xFunc, xStep, xFinal,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -102965,11 +105561,11 @@ int nName = sqlite3Strlen30(zName); int rc; sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0); + 0, sqlite3InvalidFunction, 0, 0, 0); } rc = sqlite3ApiExit(db, SQLITE_OK); sqlite3_mutex_leave(db->mutex); return rc; } @@ -103103,11 +105699,14 @@ ** registered using sqlite3_wal_hook(). Likewise, registering a callback ** using sqlite3_wal_hook() disables the automatic checkpoint mechanism ** configured by this function. */ SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifndef SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ sqlite3_wal_hook(db, 0, 0); } @@ -103191,20 +105790,11 @@ assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb && rc==SQLITE_OK; i++){ if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( sqlite3BtreeIsInReadTrans(pBt) ){ - rc = SQLITE_LOCKED; - }else{ - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); - } - } + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt); } } return rc; } @@ -103242,64 +105832,10 @@ #if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 return 0; #endif } -/* -** This routine is called to create a connection to a database BTree -** driver. If zFilename is the name of a file, then that file is -** opened and used. If zFilename is the magic name ":memory:" then -** the database is stored in memory (and is thus forgotten as soon as -** the connection is closed.) If zFilename is NULL then the database -** is a "virtual" database for transient use only and is deleted as -** soon as the connection is closed. -** -** A virtual database can be either a disk file (that is automatically -** deleted when the file is closed) or it an be held entirely in memory. -** The sqlite3TempInMemory() function is used to determine which. -*/ -SQLITE_PRIVATE int sqlite3BtreeFactory( - sqlite3 *db, /* Main database when opening aux otherwise 0 */ - const char *zFilename, /* Name of the file containing the BTree database */ - int omitJournal, /* if TRUE then do not journal this file */ - int nCache, /* How many pages in the page cache */ - int vfsFlags, /* Flags passed through to vfsOpen */ - Btree **ppBtree /* Pointer to new Btree object written here */ -){ - int btFlags = 0; - int rc; - - assert( sqlite3_mutex_held(db->mutex) ); - assert( ppBtree != 0); - if( omitJournal ){ - btFlags |= BTREE_OMIT_JOURNAL; - } - if( db->flags & SQLITE_NoReadlock ){ - btFlags |= BTREE_NO_READLOCK; - } -#ifndef SQLITE_OMIT_MEMORYDB - if( zFilename==0 && sqlite3TempInMemory(db) ){ - zFilename = ":memory:"; - } -#endif - - if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (zFilename==0 || *zFilename==0) ){ - vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; - } - rc = sqlite3BtreeOpen(zFilename, (sqlite3 *)db, ppBtree, btFlags, vfsFlags); - - /* If the B-Tree was successfully opened, set the pager-cache size to the - ** default value. Except, if the call to BtreeOpen() returned a handle - ** open on an existing shared pager-cache, do not change the pager-cache - ** size. - */ - if( rc==SQLITE_OK && 0==sqlite3BtreeSchema(*ppBtree, 0, 0) ){ - sqlite3BtreeSetCacheSize(*ppBtree, nCache); - } - return rc; -} - /* ** Return UTF-8 encoded English language explanation of the most recent ** error. */ SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ @@ -103460,17 +105996,16 @@ } } } pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl ){ - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - pColl->type = collType; - } + if( pColl==0 ) return SQLITE_NOMEM; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + pColl->type = collType; sqlite3Error(db, SQLITE_OK, 0); return SQLITE_OK; } @@ -103538,21 +106073,43 @@ ** It merely prevents new constructs that exceed the limit ** from forming. */ SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; + + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ return -1; } oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ } db->aLimit[limitId] = newLimit; } - return oldLimit; + return oldLimit; /* IMP: R-53341-35419 */ } /* ** This routine does the work of opening a database on behalf of ** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" @@ -103571,10 +106128,28 @@ *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif + + /* Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE; if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; }else if( flags & SQLITE_OPEN_NOMUTEX ){ isThreadsafe = 0; @@ -103605,11 +106180,12 @@ SQLITE_OPEN_MAIN_JOURNAL | SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_MASTER_JOURNAL | SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL ); /* Allocate the sqlite data structure */ db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; @@ -103677,13 +106253,12 @@ createCollation(db, "NOCASE", SQLITE_UTF8, SQLITE_COLL_NOCASE, 0, nocaseCollatingFunc, 0); /* Open the backend database driver */ db->openFlags = flags; - rc = sqlite3BtreeFactory(db, zFilename, 0, SQLITE_DEFAULT_CACHE_SIZE, - flags | SQLITE_OPEN_MAIN_DB, - &db->aDb[0].pBt); + rc = sqlite3BtreeOpen(zFilename, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); if( rc!=SQLITE_OK ){ if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; } sqlite3Error(db, rc, 0); @@ -104175,11 +106750,14 @@ sqlite3BtreeEnter(pBtree); pPager = sqlite3BtreePager(pBtree); assert( pPager!=0 ); fd = sqlite3PagerFile(pPager); assert( fd!=0 ); - if( fd->pMethods ){ + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( fd->pMethods ){ rc = sqlite3OsFileControl(fd, op, pArg); } sqlite3BtreeLeave(pBtree); } } @@ -104385,10 +106963,26 @@ */ case SQLITE_TESTCTRL_PGHDRSZ: { rc = sizeof(PgHdr); break; } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); + ** + ** Pass pFree into sqlite3ScratchFree(). + ** If sz>0 then allocate a scratch buffer into pNew. + */ + case SQLITE_TESTCTRL_SCRATCHMALLOC: { + void *pFree, **ppNew; + int sz; + sz = va_arg(ap, int); + ppNew = va_arg(ap, void**); + pFree = va_arg(ap, void*); + if( sz ) *ppNew = sqlite3ScratchMalloc(sz); + sqlite3ScratchFree(pFree); + break; + } } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ return rc; @@ -104649,11 +107243,11 @@ sqlite3BeginBenignMalloc(); assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3DbMallocSize(db, aDyn)/sizeof(void*))) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) ){ /* The aArg[] array needs to grow. */ void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); if( pNew ){ memcpy(pNew, aArg, nArg*sizeof(void *)); @@ -105378,12 +107972,18 @@ #ifndef SQLITE_AMALGAMATION /* ** Macros indicating that conditional expressions are always true or ** false. */ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#else # define ALWAYS(x) (x) # define NEVER(X) (x) +#endif + /* ** Internal types used by SQLite. */ typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ typedef short int i16; /* 2-byte (or larger) signed integer */ @@ -105397,12 +107997,16 @@ typedef struct Fts3Table Fts3Table; typedef struct Fts3Cursor Fts3Cursor; typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3PhraseToken Fts3PhraseToken; + typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3SegReaderArray Fts3SegReaderArray; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. @@ -105419,26 +108023,18 @@ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[25]; - - /* Pointer to string containing the SQL: - ** - ** "SELECT block FROM %_segments WHERE blockid BETWEEN ? AND ? - ** ORDER BY blockid" - */ - char *zSelectLeaves; - int nLeavesStmt; /* Valid statements in aLeavesStmt */ - int nLeavesTotal; /* Total number of prepared leaves stmts */ - int nLeavesAlloc; /* Allocated size of aLeavesStmt */ - sqlite3_stmt **aLeavesStmt; /* Array of prepared zSelectLeaves stmts */ + sqlite3_stmt *aStmt[24]; int nNodeSize; /* Soft limit for node size */ - u8 bHasContent; /* True if %_content table exists */ + u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ /* The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed @@ -105461,17 +108057,28 @@ i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ + int nMatchinfo; /* Number of elements in aMatchinfo[] */ + char *zMatchinfo; /* Matchinfo specification */ }; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 /* ** The Fts3Cursor.eSearch member is always set to one of the following. ** Actualy, Fts3Cursor.eSearch can be greater than or equal to ** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index @@ -105491,22 +108098,34 @@ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in "...", nToken will be the number -** of tokens in the string. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +** +** The nDocMatch and nMatch variables contain data that may be used by the +** matchinfo() function. They are populated when the full-text index is +** queried for hits on the phrase. If one or more tokens in the phrase +** are deferred, the nDocMatch and nMatch variables are populated based +** on the assumption that the */ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFulltext; /* True if full-text index was used */ + Fts3SegReaderArray *pArray; /* Segment-reader for this token */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ +}; + struct Fts3Phrase { + /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ int isNot; /* Phrase prefixed by unary not (-) operator */ - struct PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer pointed to by z */ - int isPrefix; /* True if token ends in with a "*" character */ - } aToken[1]; /* One entry for each token in the phrase */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** @@ -105552,32 +108171,38 @@ #define FTSQUERY_AND 3 #define FTSQUERY_OR 4 #define FTSQUERY_PHRASE 5 -/* fts3_init.c */ -SQLITE_PRIVATE int sqlite3Fts3DeleteVtab(int, sqlite3_vtab *); -SQLITE_PRIVATE int sqlite3Fts3InitVtab(int, sqlite3*, void*, int, const char*const*, - sqlite3_vtab **, char **); - /* fts3_write.c */ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64, +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); SQLITE_PRIVATE int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); SQLITE_PRIVATE int sqlite3Fts3SegReaderIterate( Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, int (*)(Fts3Table *, void *, char *, int, char *, int), void * ); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*); -SQLITE_PRIVATE int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*); +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 @@ -105597,26 +108222,28 @@ SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); SQLITE_PRIVATE char *sqlite3Fts3FindPositions(Fts3Expr *, sqlite3_int64, int); -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *, Fts3Expr *); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, - const char *, sqlite3_tokenizer **, const char **, char ** +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** ); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); /* fts3_snippet.c */ SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, const char *, const char *, int, int ); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, char **, int, int, const char *, int, Fts3Expr ** ); @@ -105761,20 +108388,17 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table *)pVtab; int i; assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); /* Free any prepared statements held */ for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } - for(i=0; inLeavesStmt; i++){ - sqlite3_finalize(p->aLeavesStmt[i]); - } - sqlite3_free(p->zSelectLeaves); - sqlite3_free(p->aLeavesStmt); + sqlite3_free(p->zSegmentsTbl); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); @@ -105781,11 +108405,11 @@ return SQLITE_OK; } /* ** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is writting +** and then evaluate those statements. The success code is written ** into *pRc. ** ** If *pRc is initially non-zero then this routine is a no-op. */ static void fts3DbExec( @@ -105833,37 +108457,42 @@ /* ** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static int fts3DeclareVtab(Fts3Table *p){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ - - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } - - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName - ); - - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(p->db, zSql); - } - - sqlite3_free(zSql); - sqlite3_free(zCols); - return rc; +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN)", zCols, p->zName + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } } /* ** Create the backing store tables (%_content, %_segments and %_segdir) ** required by the FTS3 table passed as the only argument. This is done @@ -105878,25 +108507,23 @@ int i; /* Iterator variable */ char *zContentCols; /* Columns of %_content table */ sqlite3 *db = p->db; /* The database connection */ /* Create a list of user columns for the content table */ - if( p->bHasContent ){ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); - } + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); /* Create other tables */ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", p->zDb, p->zName ); @@ -105915,52 +108542,78 @@ if( p->bHasDocsize ){ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", p->zDb, p->zName ); + } + if( p->bHasStat ){ fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_stat'(id INTEGER PRIMARY KEY, value BLOB);", p->zDb, p->zName ); } return rc; } /* -** An sqlite3_exec() callback for fts3TableExists. +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static int fts3TableExistsCallback(void *pArg, int n, char **pp1, char **pp2){ - UNUSED_PARAMETER(n); - UNUSED_PARAMETER(pp1); - UNUSED_PARAMETER(pp2); - *(int*)pArg = 1; - return 1; +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } } /* -** Determine if a table currently exists in the database. +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. */ -static void fts3TableExists( - int *pRc, /* Success code */ - sqlite3 *db, /* The database connection to test */ - const char *zDb, /* ATTACHed database within the connection */ - const char *zName, /* Name of the FTS3 table */ - const char *zSuffix, /* Shadow table extension */ - u8 *pResult /* Write results here */ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue ){ - int rc = SQLITE_OK; - int res = 0; - char *zSql; - if( *pRc ) return; - zSql = sqlite3_mprintf( - "SELECT 1 FROM %Q.sqlite_master WHERE name='%q%s'", - zDb, zName, zSuffix - ); - rc = sqlite3_exec(db, zSql, fts3TableExistsCallback, &res, 0); - sqlite3_free(zSql); - *pResult = (u8)(res & 0xff); - if( rc!=SQLITE_ABORT ) *pRc = rc; + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. @@ -105980,48 +108633,103 @@ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p; /* Pointer to allocated vtab */ - int rc; /* Return code */ + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ int nByte; /* Size of allocation used for *p */ int iCol; /* Column index */ int nString = 0; /* Bytes required to hold all column names */ int nCol = 0; /* Number of columns in the FTS table */ char *zCsr; /* Space for holding column names */ int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ - - const char *zTokenizer = 0; /* Name of tokenizer to use */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; - for(i=3; i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); } - if( z!=zTokenizer ){ + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + if( !zVal ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ + if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ + bNoDocsize = 1; + }else{ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + }else{ + *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + } + sqlite3_free(zVal); + } + + /* Otherwise, the argument is a column name. */ + else { nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; } } - nCol = argc - 3 - (zTokenizer!=0); - if( zTokenizer==0 ){ - rc = sqlite3Fts3InitTokenizer(pHash, 0, &pTokenizer, 0, pzErr); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pTokenizer ); - } + if( rc!=SQLITE_OK ) goto fts3_init_out; if( nCol==0 ){ + assert( nString==0 ); + aCol[0] = "content"; + nString = 8; nCol = 1; } + + if( pTokenizer==0 ){ + rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); + if( rc!=SQLITE_OK ) goto fts3_init_out; + } + assert( pTokenizer ); + /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ nName + /* zName */ @@ -106031,77 +108739,70 @@ if( p==0 ){ rc = SQLITE_NOMEM; goto fts3_init_out; } memset(p, 0, nByte); - p->db = db; p->nColumn = nCol; p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - zCsr = (char *)&p->azColumn[nCol]; - + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = isFts4; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->azColumn[nCol]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; p->zDb = zCsr; memcpy(zCsr, argv[1], nDb); zCsr += nDb; /* Fill in the azColumn array */ - iCol = 0; - for(i=3; iazColumn[iCol++] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } - } - if( iCol==0 ){ - assert( nCol==1 ); - p->azColumn[0] = "content"; + for(iCol=0; iColazColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); } /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ - p->bHasContent = 1; - p->bHasDocsize = argv[0][3]=='4'; rc = fts3CreateTables(p); - }else{ - rc = SQLITE_OK; - fts3TableExists(&rc, db, argv[1], argv[2], "_content", &p->bHasContent); - fts3TableExists(&rc, db, argv[1], argv[2], "_docsize", &p->bHasDocsize); - } - if( rc!=SQLITE_OK ) goto fts3_init_out; - - rc = fts3DeclareVtab(p); - if( rc!=SQLITE_OK ) goto fts3_init_out; - - *ppVTab = &p->base; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database (see + ** function sqlite3Fts3SegReaderCost() for details). + */ + fts3DatabasePageSize(&rc, p); + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); fts3_init_out: - assert( p || (pTokenizer && rc!=SQLITE_OK) ); + + sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); - }else{ + }else if( pTokenizer ){ pTokenizer->pModule->xDestroy(pTokenizer); } + }else{ + *ppVTab = &p->base; } return rc; } /* @@ -106209,14 +108910,16 @@ /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ -static int fulltextClose(sqlite3_vtab_cursor *pCursor){ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_finalize(pCsr->pStmt); sqlite3Fts3ExprFree(pCsr->pExpr); + sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); sqlite3_free(pCsr); return SQLITE_OK; } @@ -106251,134 +108954,185 @@ return SQLITE_OK; } } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. ** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ int rc = SQLITE_OK; /* Return code */ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - } - }else if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - }else{ - sqlite3_reset(pCsr->pStmt); - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - } - return rc; -} - - -/* -** The buffer pointed to by argument zNode (size nNode bytes) contains the -** root node of a b-tree segment. The segment is guaranteed to be at least -** one level high (i.e. the root node is not also a leaf). If successful, -** this function locates the leaf node of the segment that may contain the -** term specified by arguments zTerm and nTerm and writes its block number -** to *piLeaf. -** -** It is possible that the returned leaf node does not contain the specified -** term. However, if the segment does contain said term, it is stored on -** the identified leaf node. Because this function only inspects interior -** segment nodes (and never loads leaf nodes into memory), it is not possible -** to be sure. + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + int nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return SQLITE_CORRUPT; + } + + while( zCsrzEnd ){ + rc = SQLITE_CORRUPT; + goto finish_scan; + } + if( nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = (nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. ** ** If an error occurs, an error code other than SQLITE_OK is returned. */ static int fts3SelectLeaf( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ int nTerm, /* Size of term zTerm in bytes */ const char *zNode, /* Buffer containing segment interior node */ int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf /* Selected leaf node */ -){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - - while( 1 ){ - int isFirstTerm = 1; /* True when processing first term on page */ - int iHeight; /* Height of this node in tree */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ - int nBlock; /* Size of child node in bytes */ - - zCsr += sqlite3Fts3GetVarint32(zCsr, &iHeight); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - - while( zCsrnAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - sqlite3_free(zBuffer); - return SQLITE_NOMEM; - } - zBuffer = zNew; - } - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( cmp<0 || (cmp==0 && nBuffer>nTerm) ) break; - iChild++; - }; - - /* If (iHeight==1), the children of this interior node are leaves. The - ** specified term may be present on leaf node iChild. - */ - if( iHeight==1 ){ - *piLeaf = iChild; - break; - } - - /* Descend to interior node iChild. */ - rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock); - if( rc!=SQLITE_OK ) break; - zEnd = &zCsr[nBlock]; - } - sqlite3_free(zBuffer); + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + sqlite3Fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + } + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + sqlite3_free(zBlob); + } + return rc; } /* ** This function is used to create delta-encoded serialized lists of FTS3 @@ -106606,24 +109360,48 @@ *pp2 = p2 + 1; } /* ** nToken==1 searches for adjacent positions. +** +** This function is used to merge two position lists into one. When it is +** called, *pp1 and *pp2 must both point to position lists. A position-list is +** the part of a doclist that follows each document id. For example, if a row +** contains: +** +** 'a b c'|'x y z'|'a b b a' +** +** Then the position list for this row for token 'b' would consist of: +** +** 0x02 0x01 0x02 0x03 0x03 0x00 +** +** When this function returns, both *pp1 and *pp2 are left pointing to the +** byte following the 0x00 terminator of their respective position lists. +** +** If isSaveLeft is 0, an entry is added to the output position list for +** each position in *pp2 for which there exists one or more positions in +** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. */ static int fts3PoslistPhraseMerge( - char **pp, /* Output buffer */ + char **pp, /* IN/OUT: Preallocated output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ ){ char *p = (pp ? *pp : 0); char *p1 = *pp1; char *p2 = *pp2; - int iCol1 = 0; int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + assert( *p1!=0 && *p2!=0 ); if( *p1==POS_COLUMN ){ p1++; p1 += sqlite3Fts3GetVarint32(p1, &iCol1); } @@ -106648,11 +109426,13 @@ assert( *p2!=POS_END && *p2!=POS_COLUMN ); fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; while( 1 ){ - if( iPos2>iPos1 && iPos2<=iPos1+nToken ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ sqlite3_int64 iSave; if( !pp ){ fts3PoslistCopy(0, &p2); fts3PoslistCopy(0, &p1); *pp1 = p1; @@ -106731,25 +109511,25 @@ ){ char *p1 = *pp1; char *p2 = *pp2; if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, pp1, pp2) ) return 1; + if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; *pp1 = p1; *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, pp2, pp1); + return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); }else{ char *pTmp1 = aTmp; char *pTmp2; char *aTmp2; int res = 1; - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, pp1, pp2); + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); aTmp2 = pTmp2 = pTmp1; *pp1 = p1; *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, pp2, pp1); + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ fts3PoslistMerge(pp, &aTmp, &aTmp2); }else if( pTmp1!=aTmp ){ fts3PoslistCopy(pp, &aTmp); }else if( pTmp2!=aTmp2 ){ @@ -106791,11 +109571,12 @@ char *aBuffer, /* Pre-allocated output buffer */ int *pnBuffer, /* OUT: Bytes written to aBuffer */ char *a1, /* Buffer containing first doclist */ int n1, /* Size of buffer a1 */ char *a2, /* Buffer containing second doclist */ - int n2 /* Size of buffer a2 */ + int n2, /* Size of buffer a2 */ + int *pnDoc /* OUT: Number of docids in output */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; @@ -106802,10 +109583,11 @@ char *p = aBuffer; char *p1 = a1; char *p2 = a2; char *pEnd1 = &a1[n1]; char *pEnd2 = &a2[n2]; + int nDoc = 0; assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR || mergetype==MERGE_AND || mergetype==MERGE_NOT || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR @@ -106845,10 +109627,11 @@ while( p1 && p2 ){ if( i1==i2 ){ fts3PutDeltaVarint(&p, &iPrev, i1); fts3GetDeltaVarint2(&p1, pEnd1, &i1); fts3GetDeltaVarint2(&p2, pEnd2, &i2); + nDoc++; }else if( i1isReqPos ? MERGE_POS_OR : MERGE_OR); + char *aOut = 0; + int nOut = 0; + int i; + + /* Loop through the doclists in the aaOutput[] array. Merge them all + ** into a single doclist. + */ + for(i=0; iaaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew = nOut + pTS->anOutput[i]; + char *aNew = sqlite3_malloc(nNew); + if( !aNew ){ + sqlite3_free(aOut); + return SQLITE_NOMEM; + } + fts3DoclistMerge(mergetype, 0, 0, + aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + ); + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} /* ** This function is used as the sqlite3Fts3SegReaderIterate() callback when ** querying the full-text index for a doclist associated with a term or ** term-prefix. @@ -106959,44 +109793,224 @@ int nTerm, char *aDoclist, int nDoclist ){ TermSelect *pTS = (TermSelect *)pContext; - int nNew = pTS->nOutput + nDoclist; - char *aNew = sqlite3_malloc(nNew); UNUSED_PARAMETER(p); UNUSED_PARAMETER(zTerm); UNUSED_PARAMETER(nTerm); - if( !aNew ){ - return SQLITE_NOMEM; - } - - if( pTS->nOutput==0 ){ + if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). TODO: Add a way to transfer control of the ** aDoclist buffer from the caller so as to avoid the memcpy(). */ - memcpy(aNew, aDoclist, nDoclist); + pTS->aaOutput[0] = sqlite3_malloc(nDoclist); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + }else{ + return SQLITE_NOMEM; + } }else{ - /* The output buffer is not empty. Merge doclist aDoclist with the - ** existing output. This can only happen with prefix-searches (as - ** searches for exact terms return exactly one doclist). - */ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aOutput, pTS->nOutput, aDoclist, nDoclist + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + char *aNew; + int nNew; + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + } + + nNew = nMerge + pTS->anOutput[iOut]; + aNew = sqlite3_malloc(nNew); + if( !aNew ){ + if( aMerge!=aDoclist ){ + sqlite3_free(aMerge); + } + return SQLITE_NOMEM; + } + fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, + pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 + ); + + if( iOut>0 ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + return SQLITE_OK; +} + +static int fts3DeferredTermSelect( + Fts3DeferredToken *pToken, /* Phrase token */ + int isTermPos, /* True to include positions */ + int *pnOut, /* OUT: Size of list */ + char **ppOut /* OUT: Body of list */ +){ + char *aSource; + int nSource; + + aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); + if( !aSource ){ + *pnOut = 0; + *ppOut = 0; + }else if( isTermPos ){ + *ppOut = sqlite3_malloc(nSource); + if( !*ppOut ) return SQLITE_NOMEM; + memcpy(*ppOut, aSource, nSource); + *pnOut = nSource; + }else{ + sqlite3_int64 docid; + *pnOut = sqlite3Fts3GetVarint(aSource, &docid); + *ppOut = sqlite3_malloc(*pnOut); + if( !*ppOut ) return SQLITE_NOMEM; + sqlite3Fts3PutVarint(*ppOut, docid); + } + + return SQLITE_OK; +} + +/* +** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. +** Elements are added to the array using fts3SegReaderArrayAdd(). +*/ +struct Fts3SegReaderArray { + int nSegment; /* Number of valid entries in apSegment[] */ + int nAlloc; /* Allocated size of apSegment[] */ + int nCost; /* The cost of executing SegReaderIterate() */ + Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ +}; + + +/* +** Free an Fts3SegReaderArray object. Also free all seg-readers in the +** array (using sqlite3Fts3SegReaderFree()). +*/ +static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ + if( pArray ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pArray->apSegment[i]); + } + sqlite3_free(pArray); + } +} + +static int fts3SegReaderArrayAdd( + Fts3SegReaderArray **ppArray, + Fts3SegReader *pNew +){ + Fts3SegReaderArray *pArray = *ppArray; + + if( !pArray || pArray->nAlloc==pArray->nSegment ){ + int nNew = (pArray ? pArray->nAlloc+16 : 16); + pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, + sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) ); + if( !pArray ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + if( nNew==16 ){ + pArray->nSegment = 0; + pArray->nCost = 0; + } + pArray->nAlloc = nNew; + *ppArray = pArray; } - sqlite3_free(pTS->aOutput); - pTS->aOutput = aNew; - pTS->nOutput = nNew; - + pArray->apSegment[pArray->nSegment++] = pNew; return SQLITE_OK; } + +static int fts3TermSegReaderArray( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ +){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + Fts3SegReaderArray *pArray = 0; /* Array object to build */ + Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ + sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ + int iAge = 0; /* Used to assign ages to segments */ + + /* Allocate a seg-reader to scan the pending terms, if any. */ + rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); + if( rc==SQLITE_OK && pReader ) { + rc = fts3SegReaderArrayAdd(&pArray, pReader); + } + + /* Loop through the entire %_segdir table. For each segment, create a + ** Fts3SegReader to iterate through the subset of the segment leaves + ** that may contain a term that matches zTerm/nTerm. For non-prefix + ** searches, this is always a single leaf. For prefix searches, this + ** may be a contiguous block of leaves. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, &pStmt); + } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pNew = 0; + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + if( sqlite3_column_int64(pStmt, 1)==0 ){ + /* The entire segment is stored on the root node (which must be a + ** leaf). Do not bother inspecting any data in this case, just + ** create a Fts3SegReader to scan the single leaf. + */ + rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew); + }else{ + sqlite3_int64 i1; /* First leaf that may contain zTerm */ + sqlite3_int64 i2; /* Final leaf that may contain zTerm */ + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); + if( isPrefix==0 ) i2 = i1; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew); + } + } + assert( (pNew==0)==(rc!=SQLITE_OK) ); + + /* If a new Fts3SegReader was allocated, add it to the array. */ + if( rc==SQLITE_OK ){ + rc = fts3SegReaderArrayAdd(&pArray, pNew); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); + } + iAge++; + } + + if( rc==SQLITE_DONE ){ + rc = sqlite3_reset(pStmt); + }else{ + sqlite3_reset(pStmt); + } + if( rc!=SQLITE_OK ){ + fts3SegReaderArrayFree(pArray); + pArray = 0; + } + *ppArray = pArray; + return rc; +} /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** @@ -107007,140 +110021,150 @@ ** in the database without the found length specifier at the start of on-disk ** doclists. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ - int i; - TermSelect tsc; - Fts3SegFilter filter; /* Segment term filter configuration */ - Fts3SegReader **apSegment; /* Array of segments to read data from */ - int nSegment = 0; /* Size of apSegment array */ - int nAlloc = 16; /* Allocated size of segment array */ int rc; /* Return code */ - sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ - int iAge = 0; /* Used to assign ages to segments */ - - apSegment = (Fts3SegReader **)sqlite3_malloc(sizeof(Fts3SegReader*)*nAlloc); - if( !apSegment ) return SQLITE_NOMEM; - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &apSegment[0]); - if( rc!=SQLITE_OK ) goto finished; - if( apSegment[0] ){ - nSegment = 1; - } - - /* Loop through the entire %_segdir table. For each segment, create a - ** Fts3SegReader to iterate through the subset of the segment leaves - ** that may contain a term that matches zTerm/nTerm. For non-prefix - ** searches, this is always a single leaf. For prefix searches, this - ** may be a contiguous block of leaves. - ** - ** The code in this loop does not actually load any leaves into memory - ** (unless the root node happens to be a leaf). It simply examines the - ** b-tree structure to determine which leaves need to be inspected. - */ - rc = sqlite3Fts3AllSegdirs(p, &pStmt); - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pNew = 0; - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); - if( sqlite3_column_int64(pStmt, 1)==0 ){ - /* The entire segment is stored on the root node (which must be a - ** leaf). Do not bother inspecting any data in this case, just - ** create a Fts3SegReader to scan the single leaf. - */ - rc = sqlite3Fts3SegReaderNew(p, iAge, 0, 0, 0, zRoot, nRoot, &pNew); - }else{ - int rc2; /* Return value of sqlite3Fts3ReadBlock() */ - sqlite3_int64 i1; /* Blockid of leaf that may contain zTerm */ - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1); - if( rc==SQLITE_OK ){ - sqlite3_int64 i2 = sqlite3_column_int64(pStmt, 2); - rc = sqlite3Fts3SegReaderNew(p, iAge, i1, i2, 0, 0, 0, &pNew); - } - - /* The following call to ReadBlock() serves to reset the SQL statement - ** used to retrieve blocks of data from the %_segments table. If it is - ** not reset here, then it may remain classified as an active statement - ** by SQLite, which may lead to "DROP TABLE" or "DETACH" commands - ** failing. - */ - rc2 = sqlite3Fts3ReadBlock(p, 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - iAge++; - - /* If a new Fts3SegReader was allocated, add it to the apSegment array. */ - assert( pNew!=0 || rc!=SQLITE_OK ); - if( pNew ){ - if( nSegment==nAlloc ){ - Fts3SegReader **pArray; - nAlloc += 16; - pArray = (Fts3SegReader **)sqlite3_realloc( - apSegment, nAlloc*sizeof(Fts3SegReader *) - ); - if( !pArray ){ - sqlite3Fts3SegReaderFree(p, pNew); - rc = SQLITE_NOMEM; - goto finished; - } - apSegment = pArray; - } - apSegment[nSegment++] = pNew; - } - } - if( rc!=SQLITE_DONE ){ - assert( rc!=SQLITE_OK ); - goto finished; - } - + Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ + TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pArray = pTok->pArray; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY - | (isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; - filter.zTerm = zTerm; - filter.nTerm = nTerm; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; - rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, &filter, - fts3TermSelectCb, (void *)&tsc + rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, + &filter, fts3TermSelectCb, (void *)&tsc ); + if( rc==SQLITE_OK ){ + rc = fts3TermSelectMerge(&tsc); + } if( rc==SQLITE_OK ){ - *ppOut = tsc.aOutput; - *pnOut = tsc.nOutput; + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; }else{ - sqlite3_free(tsc.aOutput); + int i; + for(i=0; ipArray = 0; return rc; } +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + if( !isPoslist ){ + /* The number of docids in the list is the same as the number of + ** varints. In FTS3 a varint consists of a single byte with the 0x80 + ** bit cleared and zero or more bytes with the 0x80 bit set. So to + ** count the varints in the buffer, just count the number of bytes + ** with the 0x80 bit clear. */ + while( ppLeft); + if( rc==SQLITE_OK ){ + rc = fts3DeferExpression(pCsr, pExpr->pRight); + } + if( pExpr->eType==FTSQUERY_PHRASE ){ + int iCol = pExpr->pPhrase->iColumn; + int i; + for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ + Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; + if( pToken->pDeferred==0 ){ + rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); + } + } + } + } + return rc; +} + +/* +** This function removes the position information from a doclist. When +** called, buffer aList (size *pnList bytes) contains a doclist that includes +** position information. This function removes the position information so +** that aList contains only docids, and adjusts *pnList to reflect the new +** (possibly reduced) size of the doclist. +*/ +static void fts3DoclistStripPositions( + char *aList, /* IN/OUT: Buffer containing doclist */ + int *pnList /* IN/OUT: Size of doclist in bytes */ +){ + if( aList ){ + char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Input cursor */ + char *pOut = aList; /* Output cursor */ + + while( piColumn; int isTermPos = (pPhrase->nToken>1 || isReqPos); + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int isFirst = 1; + + int iPrevTok = 0; + int nDoc = 0; + + /* If this is an xFilter() evaluation, create a segment-reader for each + ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo + ** evaluation, only create segment-readers if there are no Fts3DeferredToken + ** objects attached to the phrase-tokens. + */ + for(ii=0; iinToken; ii++){ + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; + if( pTok->pArray==0 ){ + if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) + || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) + || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) + ){ + rc = fts3TermSegReaderArray( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + ); + if( rc!=SQLITE_OK ) return rc; + } + } + } for(ii=0; iinToken; ii++){ - struct PhraseToken *pTok = &pPhrase->aToken[ii]; - char *z = pTok->z; /* Next token of the phrase */ - int n = pTok->n; /* Size of z in bytes */ - int isPrefix = pTok->isPrefix;/* True if token is a prefix */ - char *pList; /* Pointer to token doclist */ - int nList; /* Size of buffer at pList */ - - rc = fts3TermSelect(p, iCol, z, n, isPrefix, isTermPos, &nList, &pList); + Fts3PhraseToken *pTok; /* Token to find doclist for */ + int iTok = 0; /* The token being queried this iteration */ + char *pList = 0; /* Pointer to token doclist */ + int nList = 0; /* Size of buffer at pList */ + + /* Select a token to process. If this is an xFilter() call, then tokens + ** are processed in order from least to most costly. Otherwise, tokens + ** are processed in the order in which they occur in the phrase. + */ + if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ + assert( isReqPos ); + iTok = ii; + pTok = &pPhrase->aToken[iTok]; + if( pTok->bFulltext==0 ) continue; + }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ + iTok = ii; + pTok = &pPhrase->aToken[iTok]; + }else{ + int nMinCost = 0x7FFFFFFF; + int jj; + + /* Find the remaining token with the lowest cost. */ + for(jj=0; jjnToken; jj++){ + Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; + if( pArray && pArray->nCostnCost; + } + } + pTok = &pPhrase->aToken[iTok]; + + /* This branch is taken if it is determined that loading the doclist + ** for the next token would require more IO than loading all documents + ** currently identified by doclist pOut/nOut. No further doclists will + ** be loaded from the full-text index for this phrase. + */ + if( nMinCost>nDoc && ii>0 ){ + rc = fts3DeferExpression(pCsr, pCsr->pExpr); + break; + } + } + + if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ + rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); + }else{ + if( pTok->pArray ){ + rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); + } + pTok->bFulltext = 1; + } + assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); if( rc!=SQLITE_OK ) break; - if( ii==0 ){ + if( isFirst ){ pOut = pList; nOut = nList; + if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ + nDoc = fts3DoclistCountDocids(1, pOut, nOut); + } + isFirst = 0; + iPrevTok = iTok; }else{ - /* Merge the new term list and the current output. If this is the - ** last term in the phrase, and positions are not required in the - ** output of this function, the positions can be dropped as part - ** of this merge. Either way, the result of this merge will be - ** smaller than nList bytes. The code in fts3DoclistMerge() is written - ** so that it is safe to use pList as the output as well as an input - ** in this case. + /* Merge the new term list and the current output. */ + char *aLeft, *aRight; + int nLeft, nRight; + int nDist; + int mt; + + /* If this is the final token of the phrase, and positions were not + ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. + ** This drops the position information from the output list. */ - int mergetype = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ){ - mergetype = MERGE_PHRASE; + mt = MERGE_POS_PHRASE; + if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; + + assert( iPrevTok!=iTok ); + if( iPrevToknToken ){ + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); + fts3DoclistStripPositions(pOut, &nOut); + } *paOut = pOut; *pnOut = nOut; }else{ sqlite3_free(pOut); } return rc; } +/* +** This function merges two doclists according to the requirements of a +** NEAR operator. +** +** Both input doclists must include position information. The output doclist +** includes position information if the first argument to this function +** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. +*/ static int fts3NearMerge( int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ int nNear, /* Parameter to NEAR operator */ int nTokenLeft, /* Number of tokens in LHS phrase arg */ char *aLeft, /* Doclist for LHS (incl. positions) */ @@ -107204,21 +110331,21 @@ char *aRight, /* As aLeft */ int nRight, /* As nRight */ char **paOut, /* OUT: Results of merge (malloced) */ int *pnOut /* OUT: Sized of output buffer */ ){ - char *aOut; - int rc; + char *aOut; /* Buffer to write output doclist to */ + int rc; /* Return code */ assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); aOut = sqlite3_malloc(nLeft+nRight+1); if( aOut==0 ){ rc = SQLITE_NOMEM; }else{ rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight + aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 ); if( rc!=SQLITE_OK ){ sqlite3_free(aOut); aOut = 0; } @@ -107226,21 +110353,36 @@ *paOut = aOut; return rc; } +/* +** This function is used as part of the processing for the snippet() and +** offsets() functions. +** +** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both +** have their respective doclists (including position information) loaded +** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from +** each doclist that are not within nNear tokens of a corresponding entry +** in the other doclist. +*/ SQLITE_PRIVATE int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ - int rc; + int rc; /* Return code */ + + assert( pLeft->eType==FTSQUERY_PHRASE ); + assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pLeft->isLoaded && pRight->isLoaded ); + if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ sqlite3_free(pLeft->aDoclist); sqlite3_free(pRight->aDoclist); pRight->aDoclist = 0; pLeft->aDoclist = 0; rc = SQLITE_OK; }else{ - char *aOut; - int nOut; + char *aOut; /* Buffer in which to assemble new doclist */ + int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, &aOut, &nOut @@ -107259,19 +110401,161 @@ pLeft->aDoclist = aOut; pLeft->nDoclist = nOut; } return rc; } + /* -** Evaluate the full-text expression pExpr against fts3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for +** Allocate an Fts3SegReaderArray for each token in the expression pExpr. +** The allocated objects are stored in the Fts3PhraseToken.pArray member +** variables of each token structure. +*/ +static int fts3ExprAllocateSegReaders( + Fts3Cursor *pCsr, /* FTS3 table */ + Fts3Expr *pExpr, /* Expression to create seg-readers for */ + int *pnExpr /* OUT: Number of AND'd expressions */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); + if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ + (*pnExpr)++; + pnExpr = 0; + } + + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + + for(ii=0; rc==SQLITE_OK && iinToken; ii++){ + Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; + if( pTok->pArray==0 ){ + rc = fts3TermSegReaderArray( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + ); + } + } + }else{ + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); + if( rc==SQLITE_OK ){ + rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); + } + } + return rc; +} + +/* +** Free the Fts3SegReaderArray objects associated with each token in the +** expression pExpr. In other words, this function frees the resources +** allocated by fts3ExprAllocateSegReaders(). +*/ +static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase ){ + int kk; + for(kk=0; kknToken; kk++){ + fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); + pPhrase->aToken[kk].pArray = 0; + } + } + fts3ExprFreeSegReaders(pExpr->pLeft); + fts3ExprFreeSegReaders(pExpr->pRight); + } +} + +/* +** Return the sum of the costs of all tokens in the expression pExpr. This +** function must be called after Fts3SegReaderArrays have been allocated +** for all tokens using fts3ExprAllocateSegReaders(). +*/ +static int fts3ExprCost(Fts3Expr *pExpr){ + int nCost; /* Return value */ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + nCost = 0; + for(ii=0; iinToken; ii++){ + Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray; + if( pArray ){ + nCost += pPhrase->aToken[ii].pArray->nCost; + } + } + }else{ + nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); + } + return nCost; +} + +/* +** The following is a helper function (and type) for fts3EvalExpr(). It +** must be called after Fts3SegReaders have been allocated for every token +** in the expression. See the context it is called from in fts3EvalExpr() +** for further explanation. +*/ +typedef struct ExprAndCost ExprAndCost; +struct ExprAndCost { + Fts3Expr *pExpr; + int nCost; +}; +static void fts3ExprAssignCosts( + Fts3Expr *pExpr, /* Expression to create seg-readers for */ + ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ +){ + if( pExpr->eType==FTSQUERY_AND ){ + fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); + fts3ExprAssignCosts(pExpr->pRight, ppExprCost); + }else{ + (*ppExprCost)->pExpr = pExpr; + (*ppExprCost)->nCost = fts3ExprCost(pExpr); + (*ppExprCost)++; + } +} + +/* +** Evaluate the full-text expression pExpr against FTS3 table pTab. Store +** the resulting doclist in *paOut and *pnOut. This routine mallocs for +** the space needed to store the output. The caller is responsible for ** freeing the space when it has finished. +** +** This function is called in two distinct contexts: +** +** * From within the virtual table xFilter() method. In this case, the +** output doclist contains entries for all rows in the table, based on +** data read from the full-text index. +** +** In this case, if the query expression contains one or more tokens that +** are very common, then the returned doclist may contain a superset of +** the documents that actually match the expression. +** +** * From within the virtual table xNext() method. This call is only made +** if the call from within xFilter() found that there were very common +** tokens in the query expression and did return a superset of the +** matching documents. In this case the returned doclist contains only +** entries that correspond to the current row of the table. Instead of +** reading the data for each token from the full-text index, the data is +** already available in-memory in the Fts3PhraseToken.pDeferred structures. +** See fts3EvalDeferred() for how it gets there. +** +** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is +** required) Fts3Cursor.doDeferred==1. +** +** If the SQLite invokes the snippet(), offsets() or matchinfo() function +** as part of a SELECT on an FTS3 table, this function is called on each +** individual phrase expression in the query. If there were very common tokens +** found in the xFilter() call, then this function is called once for phrase +** for each row visited, and the returned doclist contains entries for the +** current row only. Otherwise, if there were no very common tokens, then this +** function is called once only for each phrase in the query and the returned +** doclist contains entries for all rows of the table. +** +** Fts3Cursor.doDeferred==1 when this function is called on phrases as a +** result of a snippet(), offsets() or matchinfo() invocation. */ -static int evalFts3Expr( - Fts3Table *p, /* Virtual table handle */ +static int fts3EvalExpr( + Fts3Cursor *p, /* Virtual table cursor handle */ Fts3Expr *pExpr, /* Parsed fts3 expression */ char **paOut, /* OUT: Pointer to malloc'd result buffer */ int *pnOut, /* OUT: Size of buffer at *paOut */ int isReqPos /* Require positions in output buffer */ ){ @@ -107280,37 +110564,106 @@ /* Zero the output parameters. */ *paOut = 0; *pnOut = 0; if( pExpr ){ - assert( pExpr->eType==FTSQUERY_PHRASE - || pExpr->eType==FTSQUERY_NEAR - || isReqPos==0 + assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT + || pExpr->eType==FTSQUERY_PHRASE ); + assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); + if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, + rc = fts3PhraseSelect(p, pExpr->pPhrase, isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), paOut, pnOut ); + fts3ExprFreeSegReaders(pExpr); + }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ + ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ + int nExpr = 0; /* Size of aExpr[] */ + char *aRet = 0; /* Doclist to return to caller */ + int nRet = 0; /* Length of aRet[] in bytes */ + int nDoc = 0x7FFFFFFF; + + assert( !isReqPos ); + + rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); + if( rc==SQLITE_OK ){ + assert( nExpr>1 ); + aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); + if( !aExpr ) rc = SQLITE_NOMEM; + } + if( rc==SQLITE_OK ){ + int ii; /* Used to iterate through expressions */ + + fts3ExprAssignCosts(pExpr, &aExpr); + aExpr -= nExpr; + for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ + pBest = pCand; + } + } + + if( pBest->nCost>nDoc ){ + rc = fts3DeferExpression(p, p->pExpr); + break; + }else{ + rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); + if( rc!=SQLITE_OK ) break; + pBest->pExpr = 0; + if( ii==0 ){ + aRet = aNew; + nRet = nNew; + nDoc = fts3DoclistCountDocids(0, aRet, nRet); + }else{ + fts3DoclistMerge( + MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc + ); + sqlite3_free(aNew); + } + } + } + } + + if( rc==SQLITE_OK ){ + *paOut = aRet; + *pnOut = nRet; + }else{ + assert( *paOut==0 ); + sqlite3_free(aRet); + } + sqlite3_free(aExpr); + fts3ExprFreeSegReaders(pExpr); + }else{ char *aLeft; char *aRight; int nLeft; int nRight; - if( 0==(rc = evalFts3Expr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = evalFts3Expr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) + assert( pExpr->eType==FTSQUERY_NEAR + || pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_NOT + || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) + ); + + if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) + && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - ); switch( pExpr->eType ){ case FTSQUERY_NEAR: { Fts3Expr *pLeft; Fts3Expr *pRight; - int mergetype = isReqPos ? MERGE_POS_NEAR : MERGE_NEAR; - + int mergetype = MERGE_NEAR; if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ mergetype = MERGE_POS_NEAR; } pLeft = pExpr->pLeft; while( pLeft->eType==FTSQUERY_NEAR ){ @@ -107335,21 +110688,21 @@ ** so that a buffer of zero bytes is never allocated - this can ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. */ char *aBuffer = sqlite3_malloc(nRight+nLeft+1); rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aBuffer; sqlite3_free(aLeft); break; } default: { assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight + aLeft, nLeft, aRight, nRight, 0 ); *paOut = aLeft; break; } } @@ -107356,10 +110709,93 @@ } sqlite3_free(aRight); } } + assert( rc==SQLITE_OK || *paOut==0 ); + return rc; +} + +/* +** This function is called from within xNext() for each row visited by +** an FTS3 query. If evaluating the FTS3 query expression within xFilter() +** was able to determine the exact set of matching rows, this function sets +** *pbRes to true and returns SQLITE_IO immediately. +** +** Otherwise, if evaluating the query expression within xFilter() returned a +** superset of the matching documents instead of an exact set (this happens +** when the query includes very common tokens and it is deemed too expensive to +** load their doclists from disk), this function tests if the current row +** really does match the FTS3 query. +** +** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK +** is returned and *pbRes is set to true if the current row matches the +** FTS3 query (and should be included in the results returned to SQLite), or +** false otherwise. +*/ +static int fts3EvalDeferred( + Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ + int *pbRes /* OUT: Set to true if row is a match */ +){ + int rc = SQLITE_OK; + if( pCsr->pDeferred==0 ){ + *pbRes = 1; + }else{ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + sqlite3Fts3FreeDeferredDoclists(pCsr); + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + if( rc==SQLITE_OK ){ + char *a = 0; + int n = 0; + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); + assert( n>=0 ); + *pbRes = (n>0); + sqlite3_free(a); + } + } + return rc; +} + +/* +** Advance the cursor to the next row in the %_content table that +** matches the search criteria. For a MATCH search, this will be +** the next row that matches. For a full-table scan, this will be +** simply the next row in the %_content table. For a docid lookup, +** this routine simply sets the EOF flag. +** +** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned +** even if we reach end-of-file. The fts3EofMethod() will be called +** subsequently to determine whether or not an EOF was hit. +*/ +static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ + int res; + int rc = SQLITE_OK; /* Return code */ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + pCsr->eEvalmode = FTS3_EVAL_NEXT; + do { + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + break; + } + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + }else{ + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + break; + } + sqlite3_reset(pCsr->pStmt); + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + } + }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + return rc; } /* ** This is the xFilter interface for the virtual table. See @@ -107375,15 +110811,10 @@ ** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The ** column on the left-hand side of the MATCH operator is column ** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand ** side of the MATCH operator. */ -/* TODO(shess) Upgrade the cursor initialization and destruction to -** account for fts3FilterMethod() being called multiple times on the -** same cursor. The current solution is very fragile. Apply fix to -** fts3 as appropriate. -*/ static int fts3FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ @@ -107402,35 +110833,19 @@ UNUSED_PARAMETER(nVal); assert( idxNum>=0 && idxNum<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( nVal==0 || nVal==1 ); assert( (nVal==0)==(idxNum==FTS3_FULLSCAN_SEARCH) ); + assert( p->pSegments==0 ); /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ) return rc; - pCsr->eSearch = (i16)idxNum; - - if( idxNum==FTS3_DOCID_SEARCH ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); - }else if( idxNum!=FTS3_FULLSCAN_SEARCH ){ + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); if( zQuery==0 && sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ return SQLITE_NOMEM; @@ -107445,14 +110860,36 @@ zQuery); } return rc; } - rc = evalFts3Expr(p, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + rc = sqlite3Fts3ReadLock(p); + if( rc!=SQLITE_OK ) return rc; + + rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } + if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + } + pCsr->eSearch = (i16)idxNum; if( rc!=SQLITE_OK ) return rc; return fts3NextMethod(pCursor); } @@ -107473,10 +110910,15 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ + /* This branch runs if the query is implemented using a full-table scan + ** (not using the full-text index). In this case grab the rowid from the + ** SELECT statement. + */ + assert( pCsr->isRequireSeek==0 ); *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); } return SQLITE_OK; } @@ -107535,11 +110977,13 @@ /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - return sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); + sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + return rc; } /* ** Implementation of xBegin() method. This is a no-op. */ @@ -107573,12 +111017,31 @@ ** Load the doclist associated with expression pExpr to pExpr->aDoclist. ** The loaded doclist contains positions as well as the document ids. ** This is used by the matchinfo(), snippet() and offsets() auxillary ** functions. */ -SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Table *pTab, Fts3Expr *pExpr){ - return evalFts3Expr(pTab, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); +SQLITE_PRIVATE int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc; + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3ExprLoadFtDoclist( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + char **paDoclist, + int *pnDoclist +){ + int rc; + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; + rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); + pCsr->eEvalmode = FTS3_EVAL_NEXT; + return rc; } /* ** After ExprLoadDoclist() (see above) has been called, this function is ** used to iterate/search through the position lists that make up the doclist @@ -107640,11 +111103,11 @@ */ static int fts3FunctionArg( sqlite3_context *pContext, /* SQL function call context */ const char *zFunc, /* Function name */ sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ Fts3Cursor *pRet; if( sqlite3_value_type(pVal)!=SQLITE_BLOB || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) ){ @@ -107766,19 +111229,17 @@ sqlite3_context *pContext, /* SQLite function call context */ int nVal, /* Size of argument array */ sqlite3_value **apVal /* Array of arguments */ ){ Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - - if( nVal!=1 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function matchinfo()", -1); - return; - } - + assert( nVal==1 || nVal==2 ); if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - sqlite3Fts3Matchinfo(pContext, pCsr); + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } } /* ** This routine implements the xFindFunction method for the FTS3 @@ -107823,25 +111284,29 @@ static int fts3RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db; /* Database connection */ + sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ - - db = p->db; - rc = SQLITE_OK; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ){ + return rc; + } + fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", p->zDb, p->zName, zName ); - if( rc==SQLITE_ERROR ) rc = SQLITE_OK; if( p->bHasDocsize ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", p->zDb, p->zName, zName ); + } + if( p->bHasStat ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", p->zDb, p->zName, zName ); } @@ -107862,11 +111327,11 @@ /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, /* xDisconnect */ fts3DisconnectMethod, /* xDestroy */ fts3DestroyMethod, /* xOpen */ fts3OpenMethod, - /* xClose */ fulltextClose, + /* xClose */ fts3CloseMethod, /* xFilter */ fts3FilterMethod, /* xNext */ fts3NextMethod, /* xEof */ fts3EofMethod, /* xColumn */ fts3ColumnMethod, /* xRowid */ fts3RowidMethod, @@ -107889,23 +111354,24 @@ sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } /* -** The fts3 built-in tokenizers - "simple" and "porter" - are implemented -** in files fts3_tokenizer1.c and fts3_porter.c respectively. The following -** two forward declarations are for functions declared in these files -** used to retrieve the respective implementations. +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. ** ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point a the "simple" tokenizer implementation. -** Function ...PorterTokenizerModule() sets *pModule to point to the -** porter tokenizer/stemmer implementation. +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. */ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifdef SQLITE_ENABLE_ICU SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif /* ** Initialise the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this @@ -107957,11 +111423,12 @@ */ if( SQLITE_OK==rc && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); @@ -108097,12 +111564,24 @@ ** is defined to accept an argument of type char, and always returns 0 for ** any values that fall outside of the range of the unsigned char type (i.e. ** negative values). */ static int fts3isspace(char c){ - return (c&0x80)==0 ? isspace(c) : 0; + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(int nByte){ + void *pRet = sqlite3_malloc(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + /* ** Extract the next token from buffer z (length n) using the tokenizer ** and other information (column names etc.) in pParse. Create an Fts3Expr ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this @@ -108137,15 +111616,14 @@ pCursor->pTokenizer = pTokenizer; rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)sqlite3_malloc(nByte); + pRet = (Fts3Expr *)fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); pRet->eType = FTSQUERY_PHRASE; pRet->pPhrase = (Fts3Phrase *)&pRet[1]; pRet->pPhrase->nToken = 1; pRet->pPhrase->iColumn = iCol; pRet->pPhrase->aToken[0].n = nToken; @@ -108217,20 +111695,21 @@ const char *zToken; int nToken, iBegin, iEnd, iPos; rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(struct PhraseToken)); + p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); if( !p || !zTemp ){ goto no_mem; } if( ii==0 ){ memset(p, 0, nByte); p->pPhrase = (Fts3Phrase *)&p[1]; } p->pPhrase = (Fts3Phrase *)&p[1]; + memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); p->pPhrase->nToken = ii+1; p->pPhrase->aToken[ii].n = nToken; memcpy(&zTemp[nTemp], zToken, nToken); nTemp += nToken; if( iEndpPhrase->nToken-1):0) * sizeof(struct PhraseToken); + nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); p = fts3ReallocOrFree(p, nByte + nTemp); if( !p ){ goto no_mem; } if( zTemp ){ @@ -108366,15 +111845,14 @@ */ cNext = zInput[nKey]; if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ - pRet = (Fts3Expr *)sqlite3_malloc(sizeof(Fts3Expr)); + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); if( !pRet ){ return SQLITE_NOMEM; } - memset(pRet, 0, sizeof(Fts3Expr)); pRet->eType = pKey->eType; pRet->nNear = nNear; *ppExpr = pRet; *pnConsumed = (int)((zInput - z) + nKey); return SQLITE_OK; @@ -108388,11 +111866,10 @@ /* Check for an open bracket. */ if( sqlite3_fts3_enable_parentheses ){ if( *zInput=='(' ){ int nConsumed; - int rc; pParse->nNest++; rc = fts3ExprParse(pParse, &zInput[1], nInput-1, ppExpr, &nConsumed); if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } @@ -108546,17 +112023,16 @@ if( !sqlite3_fts3_enable_parentheses && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot ){ /* Create an implicit NOT operator. */ - Fts3Expr *pNot = sqlite3_malloc(sizeof(Fts3Expr)); + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - memset(pNot, 0, sizeof(Fts3Expr)); pNot->eType = FTSQUERY_NOT; pNot->pRight = p; if( pNotBranch ){ pNot->pLeft = pNotBranch; } @@ -108580,17 +112056,16 @@ if( isPhrase && !isRequirePhrase ){ /* Insert an implicit AND operator. */ Fts3Expr *pAnd; assert( pRet && pPrev ); - pAnd = sqlite3_malloc(sizeof(Fts3Expr)); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); if( !pAnd ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; goto exprparse_out; } - memset(pAnd, 0, sizeof(Fts3Expr)); pAnd->eType = FTSQUERY_AND; insertBinaryOperator(&pRet, pPrev, pAnd); pPrev = pAnd; } @@ -108770,51 +112245,57 @@ return sqlite3_finalize(pStmt); } /* -** This function is part of the test interface for the query parser. It -** writes a text representation of the query expression pExpr into the -** buffer pointed to by argument zBuf. It is assumed that zBuf is large -** enough to store the required text representation. +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). */ -static void exprToString(Fts3Expr *pExpr, char *zBuf){ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; int i; - zBuf += sprintf(zBuf, "PHRASE %d %d", pPhrase->iColumn, pPhrase->isNot); - for(i=0; inToken; i++){ - zBuf += sprintf(zBuf," %.*s",pPhrase->aToken[i].n,pPhrase->aToken[i].z); - zBuf += sprintf(zBuf,"%s", (pPhrase->aToken[i].isPrefix?"+":"")); + zBuf = sqlite3_mprintf( + "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); } - return; + return zBuf; } case FTSQUERY_NEAR: - zBuf += sprintf(zBuf, "NEAR/%d ", pExpr->nNear); + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); break; case FTSQUERY_NOT: - zBuf += sprintf(zBuf, "NOT "); + zBuf = sqlite3_mprintf("%zNOT ", zBuf); break; case FTSQUERY_AND: - zBuf += sprintf(zBuf, "AND "); + zBuf = sqlite3_mprintf("%zAND ", zBuf); break; case FTSQUERY_OR: - zBuf += sprintf(zBuf, "OR "); + zBuf = sqlite3_mprintf("%zOR ", zBuf); break; } - zBuf += sprintf(zBuf, "{"); - exprToString(pExpr->pLeft, zBuf); - zBuf += strlen(zBuf); - zBuf += sprintf(zBuf, "} "); - - zBuf += sprintf(zBuf, "{"); - exprToString(pExpr->pRight, zBuf); - zBuf += strlen(zBuf); - zBuf += sprintf(zBuf, "}"); + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; } /* ** This is the implementation of a scalar SQL function used to test the ** expression parser. It should be called as follows: @@ -108841,10 +112322,11 @@ const char *zExpr; int nExpr; int nCol; int ii; Fts3Expr *pExpr; + char *zBuf = 0; sqlite3 *db = sqlite3_context_db_handle(context); if( argc<3 ){ sqlite3_result_error(context, "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 @@ -108883,21 +112365,20 @@ } rc = sqlite3Fts3ExprParse( pTokenizer, azCol, nCol, nCol, zExpr, nExpr, &pExpr ); - if( rc==SQLITE_NOMEM ){ + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3_result_error(context, "Error parsing expression", -1); + }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); - goto exprtest_out; - }else if( rc==SQLITE_OK ){ - char zBuf[4096]; - exprToString(pExpr, zBuf); + }else{ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); - sqlite3Fts3ExprFree(pExpr); - }else{ - sqlite3_result_error(context, "Error parsing expression", -1); + sqlite3_free(zBuf); } + + sqlite3Fts3ExprFree(pExpr); exprtest_out: if( pModule && pTokenizer ){ rc = pModule->xDestroy(pTokenizer); } @@ -109637,11 +113118,11 @@ */ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ int i, j; char zReverse[28]; char *z, *z2; - if( nIn<3 || nIn>=sizeof(zReverse)-7 ){ + if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){ /* The word is too big or too small for the porter stemmer. ** Fallback to the copy stemmer */ copy_stemmer(zIn, nIn, zOut, pnOut); return; } @@ -110036,11 +113517,11 @@ } sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); } -static int fts3IsIdChar(char c){ +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ static const char isFtsIdChar[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ @@ -110074,13 +113555,13 @@ while( *z2 && z2[0]!=']' ) z2++; if( *z2 ) z2++; break; default: - if( fts3IsIdChar(*z1) ){ + if( sqlite3Fts3IsIdChar(*z1) ){ z2 = &z1[1]; - while( fts3IsIdChar(*z2) ) z2++; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; }else{ z1++; } } } @@ -110089,42 +113570,30 @@ return z1; } SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( Fts3Hash *pHash, /* Tokenizer hash table */ - const char *zArg, /* Possible tokenizer specification */ + const char *zArg, /* Tokenizer name */ sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ - const char **pzTokenizer, /* OUT: Set to zArg if is tokenizer */ char **pzErr /* OUT: Set to malloced error message */ ){ int rc; char *z = (char *)zArg; int n; char *zCopy; char *zEnd; /* Pointer to nul-term of zCopy */ sqlite3_tokenizer_module *m; - if( !z ){ - zCopy = sqlite3_mprintf("simple"); - }else{ - if( sqlite3_strnicmp(z, "tokenize", 8) || fts3IsIdChar(z[8])){ - return SQLITE_OK; - } - zCopy = sqlite3_mprintf("%s", &z[8]); - *pzTokenizer = zArg; - } - if( !zCopy ){ - return SQLITE_NOMEM; - } - + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; zEnd = &zCopy[strlen(zCopy)]; z = (char *)sqlite3Fts3NextToken(zCopy, &n); z[n] = '\0'; sqlite3Fts3Dequote(z); - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, z, (int)strlen(z)+1); + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); if( !m ){ *pzErr = sqlite3_mprintf("unknown tokenizer: %s", z); rc = SQLITE_ERROR; }else{ char const **aArg = 0; @@ -110414,19 +113883,27 @@ if( !zTest || !zTest2 ){ rc = SQLITE_NOMEM; } #endif - if( SQLITE_OK!=rc - || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0)) + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0); + } #ifdef SQLITE_TEST - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0)) - || SQLITE_OK!=(rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0)) + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } #endif - ); #ifdef SQLITE_TEST sqlite3_free(zTest); sqlite3_free(zTest2); #endif @@ -110484,10 +113961,13 @@ static int simpleDelim(simple_tokenizer *t, unsigned char c){ return c<0x80 && t->delim[c]; } +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} /* ** Create a new tokenizer instance. */ static int simpleCreate( @@ -110518,11 +113998,11 @@ } } else { /* Mark non-alphanumeric ASCII characters as delimiters */ int i; for(i=1; i<0x80; i++){ - t->delim[i] = !isalnum(i) ? -1 : 0; + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; } } *ppTokenizer = &t->base; return SQLITE_OK; @@ -110624,11 +114104,11 @@ for(i=0; ipToken[i] = (char)(ch<0x80 ? tolower(ch) : ch); + c->pToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); } *ppToken = c->pToken; *pnBytes = n; *piStartOffset = iStartOffset; *piEndOffset = c->iOffset; @@ -110686,10 +114166,22 @@ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* @@ -110704,19 +114196,31 @@ sqlite3_int64 iLastDocid; sqlite3_int64 iLastCol; sqlite3_int64 iLastPos; }; + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + /* ** An instance of this structure is used to iterate through the terms on ** a contiguous set of segment b-tree leaf nodes. Although the details of ** this structure are only manipulated by code in this file, opaque handles ** of type Fts3SegReader* are also used by code in fts3.c to iterate through ** terms when querying the full-text index. See functions: ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: ** ** fts3SegReaderNext() @@ -110723,34 +114227,38 @@ ** fts3SegReaderFirstDocid() ** fts3SegReaderNextDocid() */ struct Fts3SegReader { int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ - sqlite3_int64 iStartBlock; - sqlite3_int64 iEndBlock; - sqlite3_stmt *pStmt; /* SQL Statement to access leaf nodes */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ - int nTermAlloc; /* Allocated size of zTerm buffer */ Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly ** by the caller. They are valid from the time SegmentReaderNew() returns ** until SegmentReaderNext() returns something other than SQLITE_OK ** (i.e. SQLITE_DONE). */ int nTerm; /* Number of bytes in current term */ char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ /* The following variables are used to iterate through the current doclist */ char *pOffsetList; sqlite3_int64 iDocid; }; #define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->aNode==(char *)&(p)[1]) /* ** An instance of this structure is used to create a segment b-tree in the ** database. The internal details of this type are only accessed by the ** following functions: @@ -110815,16 +114323,15 @@ #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_COUNT_MAX 15 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 -#define SQL_GET_BLOCK 19 -#define SQL_DELETE_DOCSIZE 20 -#define SQL_REPLACE_DOCSIZE 21 -#define SQL_SELECT_DOCSIZE 22 -#define SQL_SELECT_DOCTOTAL 23 -#define SQL_REPLACE_DOCTOTAL 24 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_DOCTOTAL 22 +#define SQL_REPLACE_DOCTOTAL 23 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. @@ -110865,16 +114372,15 @@ /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", /* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", -/* 19 */ "SELECT block FROM %Q.'%q_segments' WHERE blockid = ?", -/* 20 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", -/* 21 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", -/* 22 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 23 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", -/* 24 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); @@ -110918,10 +114424,55 @@ } } *pp = pStmt; return rc; } + +static int fts3SelectDocsize( + Fts3Table *pTab, /* FTS3 table handle */ + int eStmt, /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */ + sqlite3_int64 iDocid, /* Docid to bind for SQL_SELECT_DOCSIZE */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + sqlite3_stmt *pStmt = 0; /* Statement requested from fts3SqlStmt() */ + int rc; /* Return code */ + + assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL ); + + rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( eStmt==SQL_SELECT_DOCSIZE ){ + sqlite3_bind_int64(pStmt, 1, iDocid); + } + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; + pStmt = 0; + }else{ + rc = SQLITE_OK; + } + } + + *ppStmt = pStmt; + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt); +} + +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_int64 iDocid, /* Docid to read size data for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt); +} /* ** Similar to fts3SqlStmt(). Except, after binding the parameters in ** array apVal[] to the SQL statement identified by eStmt, the statement ** is executed. @@ -110946,46 +114497,37 @@ *pRC = rc; } /* -** Read a single block from the %_segments table. If the specified block -** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO -** etc.) occurs, return the appropriate SQLite error code. -** -** Otherwise, if successful, set *pzBlock to point to a buffer containing -** the block read from the database, and *pnBlock to the size of the read -** block in bytes. -** -** WARNING: The returned buffer is only valid until the next call to -** sqlite3Fts3ReadBlock(). -*/ -SQLITE_PRIVATE int sqlite3Fts3ReadBlock( - Fts3Table *p, - sqlite3_int64 iBlock, - char const **pzBlock, - int *pnBlock -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_reset(pStmt); - - if( pzBlock ){ - sqlite3_bind_int64(pStmt, 1, iBlock); - rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ - return (rc==SQLITE_DONE ? SQLITE_CORRUPT : rc); - } - - *pnBlock = sqlite3_column_bytes(pStmt, 0); - *pzBlock = (char *)sqlite3_column_blob(pStmt, 0); - if( sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ - return SQLITE_CORRUPT; - } - } - return SQLITE_OK; +** This function ensures that the caller has obtained a shared-cache +** table-lock on the %_content table. This is required before reading +** data from the fts3 table. If this lock is not acquired first, then +** the caller may end up holding read-locks on the %_segments and %_segdir +** tables, but no read-lock on the %_content table. If this happens +** a second connection will be able to write to the fts3 table, but +** attempting to commit those writes might return SQLITE_LOCKED or +** SQLITE_LOCKED_SHAREDCACHE (because the commit attempts to obtain +** write-locks on the %_segments and %_segdir ** tables). +** +** We try to avoid this because if FTS3 returns any error when committing +** a transaction, the whole transaction will be rolled back. And this is +** not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. It can +** still happen if the user reads data directly from the %_segments or +** %_segdir tables instead of going through FTS3 though. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadLock(Fts3Table *p){ + int rc; /* Return code */ + sqlite3_stmt *pStmt; /* Statement used to obtain lock */ + + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + return rc; } /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, @@ -111122,14 +114664,14 @@ ** p->iPrevDocid, and the column is specified by argument iCol. ** ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ static int fts3PendingTermsAdd( - Fts3Table *p, /* FTS table into which text will be inserted */ - const char *zText, /* Text of document to be inseted */ - int iCol, /* Column number into which text is inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + Fts3Table *p, /* Table into which text will be inserted */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* OUT: Number of tokens inserted */ ){ int rc; int iStart; int iEnd; int iPos; @@ -111210,10 +114752,13 @@ } p->iPrevDocid = iDocid; return SQLITE_OK; } +/* +** Discard the contents of the pending-terms hash table. +*/ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ sqlite3_free(fts3HashData(pElem)); } @@ -111237,10 +114782,11 @@ int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]); if( rc!=SQLITE_OK ){ return rc; } } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } /* @@ -111324,10 +114870,12 @@ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); } return rc; } @@ -111334,11 +114882,11 @@ /* ** The first element in the apVal[] array is assumed to contain the docid ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ -static void fts3DeleteTerms( +static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value **apVal, /* apVal[] contains the docid to be deleted */ u32 *aSz /* Sizes of deleted document written here */ ){ @@ -111356,10 +114904,11 @@ if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); *pRC = rc; return; } + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); } } rc = sqlite3_reset(pSelect); }else{ sqlite3_reset(pSelect); @@ -111417,17 +114966,98 @@ } } return rc; } + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob /* OUT: Size of blob data */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + *pnBlob = nByte; + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3SegReader *pReader){ +static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ if( !pReader->aDoclist ){ @@ -111435,11 +115065,12 @@ }else{ pNext = &pReader->aDoclist[pReader->nDoclist]; } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; + int rc; /* Return code from Fts3ReadBlock() */ + if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); if( pElem==0 ){ pReader->aNode = 0; }else{ @@ -111451,26 +115082,40 @@ pReader->ppNextElem++; assert( pReader->aNode ); } return SQLITE_OK; } - if( !pReader->pStmt ){ - pReader->aNode = 0; + + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + pReader->aNode = 0; + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ return SQLITE_OK; } - rc = sqlite3_step(pReader->pStmt); - if( rc!=SQLITE_ROW ){ - pReader->aNode = 0; - return (rc==SQLITE_DONE ? SQLITE_OK : rc); - } - pReader->nNode = sqlite3_column_bytes(pReader->pStmt, 0); - pReader->aNode = (char *)sqlite3_column_blob(pReader->pStmt, 0); + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + ); + if( rc!=SQLITE_OK ) return rc; pNext = pReader->aNode; } + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. + */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); + if( nPrefix<0 || nSuffix<=0 + || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + ){ + return SQLITE_CORRUPT; + } if( nPrefix+nSuffix>pReader->nTermAlloc ){ int nNew = (nPrefix+nSuffix)*2; char *zNew = sqlite3_realloc(pReader->zTerm, nNew); if( !zNew ){ @@ -111481,13 +115126,22 @@ } memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; pNext += sqlite3Fts3GetVarint32(pNext, &pReader->nDoclist); - assert( pNext<&pReader->aNode[pReader->nNode] ); pReader->aDoclist = pNext; pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + || pReader->aDoclist[pReader->nDoclist-1] + ){ + return SQLITE_CORRUPT; + } return SQLITE_OK; } /* ** Set the SegReader to point to the first docid in the doclist associated @@ -111545,37 +115199,111 @@ sqlite3_int64 iDelta; pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); pReader->iDocid += iDelta; } } + +/* +** This function is called to estimate the amount of data that will be +** loaded from the disk If SegReaderIterate() is called on this seg-reader, +** in units of average document size. +** +** This can be used as follows: If the caller has a small doclist that +** contains references to N documents, and is considering merging it with +** a large doclist (size X "average documents"), it may opt not to load +** the large doclist if X>N. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCost( + Fts3Cursor *pCsr, /* FTS3 cursor handle */ + Fts3SegReader *pReader, /* Segment-reader handle */ + int *pnCost /* IN/OUT: Number of bytes read */ +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Return code */ + int nCost = 0; /* Cost in bytes to return */ + int pgsz = p->nPgsz; /* Database page size */ + + /* If this seg-reader is reading the pending-terms table, or if all data + ** for the segment is stored on the root page of the b-tree, then the cost + ** is zero. In this case all required data is already in main memory. + */ + if( p->bHasStat + && !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + int nBlob = 0; + sqlite3_int64 iBlock; + + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); + if( rc ) return rc; + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *a = sqlite3_column_blob(pStmt, 0); + if( a ){ + const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anRowAvg = (int)(((nByte / nDoc) + pgsz - 1) / pgsz); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK || pCsr->nRowAvg==0 ) return rc; + } + + /* Assume that a blob flows over onto overflow pages if it is larger + ** than (pgsz-35) bytes in size (the file-format documentation + ** confirms this). + */ + for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + int nOvfl = (nBlob + 34)/pgsz; + nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); + } + } + } + + *pnCost += nCost; + return rc; +} /* ** Free all allocations associated with the iterator passed as the ** second argument. */ -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3Table *p, Fts3SegReader *pReader){ - if( pReader ){ - if( pReader->pStmt ){ - /* Move the leaf-range SELECT statement to the aLeavesStmt[] array, - ** so that it can be reused when required by another query. - */ - assert( p->nLeavesStmtnLeavesTotal ); - sqlite3_reset(pReader->pStmt); - p->aLeavesStmt[p->nLeavesStmt++] = pReader->pStmt; - } - if( !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); - } - sqlite3_free(pReader); - } +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader && !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + } + sqlite3_free(pReader); } /* ** Allocate a new SegReader object. */ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( - Fts3Table *p, /* Virtual table handle */ int iAge, /* Segment "age". */ sqlite3_int64 iStartLeaf, /* First leaf to traverse */ sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ sqlite3_int64 iEndBlock, /* Final block of segment */ const char *zRoot, /* Buffer containing root node */ @@ -111584,81 +115312,39 @@ ){ int rc = SQLITE_OK; /* Return code */ Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ + assert( iStartLeaf<=iEndLeaf ); if( iStartLeaf==0 ){ - nExtra = nRoot; + nExtra = nRoot + FTS3_NODE_PADDING; } pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } memset(pReader, 0, sizeof(Fts3SegReader)); - pReader->iStartBlock = iStartLeaf; pReader->iIdx = iAge; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; pReader->iEndBlock = iEndBlock; if( nExtra ){ /* The entire segment is stored in the root node. */ pReader->aNode = (char *)&pReader[1]; pReader->nNode = nRoot; memcpy(pReader->aNode, zRoot, nRoot); - }else{ - /* If the text of the SQL statement to iterate through a contiguous - ** set of entries in the %_segments table has not yet been composed, - ** compose it now. - */ - if( !p->zSelectLeaves ){ - p->zSelectLeaves = sqlite3_mprintf( - "SELECT block FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ? " - "ORDER BY blockid", p->zDb, p->zName - ); - if( !p->zSelectLeaves ){ - rc = SQLITE_NOMEM; - goto finished; - } - } - - /* If there are no free statements in the aLeavesStmt[] array, prepare - ** a new statement now. Otherwise, reuse a prepared statement from - ** aLeavesStmt[]. - */ - if( p->nLeavesStmt==0 ){ - if( p->nLeavesTotal==p->nLeavesAlloc ){ - int nNew = p->nLeavesAlloc + 16; - sqlite3_stmt **aNew = (sqlite3_stmt **)sqlite3_realloc( - p->aLeavesStmt, nNew*sizeof(sqlite3_stmt *) - ); - if( !aNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - p->nLeavesAlloc = nNew; - p->aLeavesStmt = aNew; - } - rc = sqlite3_prepare_v2(p->db, p->zSelectLeaves, -1, &pReader->pStmt, 0); - if( rc!=SQLITE_OK ){ - goto finished; - } - p->nLeavesTotal++; - }else{ - pReader->pStmt = p->aLeavesStmt[--p->nLeavesStmt]; - } - - /* Bind the start and end leaf blockids to the prepared SQL statement. */ - sqlite3_bind_int64(pReader->pStmt, 1, iStartLeaf); - sqlite3_bind_int64(pReader->pStmt, 2, iEndLeaf); - } - rc = fts3SegReaderNext(pReader); - - finished: + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + if( rc==SQLITE_OK ){ *ppReader = pReader; }else{ - sqlite3Fts3SegReaderFree(p, pReader); + sqlite3Fts3SegReaderFree(pReader); } return rc; } /* @@ -111745,11 +115431,10 @@ }else{ memset(pReader, 0, nByte); pReader->iIdx = 0x7FFFFFFF; pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); - fts3SegReaderNext(pReader); } } if( isPrefix ){ sqlite3_free(aElem); @@ -111778,16 +115463,15 @@ ** If successful, the Fts3SegReader is left pointing to the first term ** in the segment and SQLITE_OK is returned. Otherwise, an SQLite error ** code is returned. */ static int fts3SegReaderNew( - Fts3Table *p, /* Virtual table handle */ sqlite3_stmt *pStmt, /* See above */ int iAge, /* Segment "age". */ Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ ){ - return sqlite3Fts3SegReaderNew(p, iAge, + return sqlite3Fts3SegReaderNew(iAge, sqlite3_column_int64(pStmt, 1), sqlite3_column_int64(pStmt, 2), sqlite3_column_int64(pStmt, 3), sqlite3_column_blob(pStmt, 4), sqlite3_column_bytes(pStmt, 4), @@ -111987,11 +115671,11 @@ /* ** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger ** (according to memcmp) than the previous term. */ static int fts3NodeAddTerm( - Fts3Table *p, /* Virtual table handle */ + Fts3Table *p, /* Virtual table handle */ SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ int isCopyTerm, /* True if zTerm/nTerm is transient */ const char *zTerm, /* Pointer to buffer containing term */ int nTerm /* Size of term in bytes */ ){ @@ -112617,19 +116301,18 @@ ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - if( pFilter->zTerm ){ + for(i=0; inTerm; const char *zTerm = pFilter->zTerm; - for(i=0; iaNode ){ int nTerm = apSegment[0]->nTerm; @@ -112734,11 +116417,11 @@ if( pFilter->zTerm && !isPrefix ){ goto finished; } for(i=0; ibase.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0); - if( rc ){ - return rc; - } - sqlite3_bind_int64(pStmt, 1, pCur->iPrevId); - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - for(i=j=0; inColumn && jbase.pVtab; - rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); - if( rc ){ - return rc; - } - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - nBlob = sqlite3_column_bytes(pStmt, 0); - pBlob = (const char*)sqlite3_column_blob(pStmt, 0); - j = sqlite3Fts3GetVarint(pBlob, &x); - a[0] = nDoc = (u32)(x & 0xffffffff); - for(i=0; inColumn && jiPrevDocid. The sizes are encoded as ** a blob of varints. */ @@ -113049,64 +116663,76 @@ sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); } /* -** Update the 0 record of the %_stat table so that it holds a blob -** which contains the document count followed by the cumulative -** document sizes for all columns. +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** */ static void fts3UpdateDocTotals( - int *pRC, /* The result code */ - Fts3Table *p, /* Table being updated */ - u32 *aSzIns, /* Size increases */ - u32 *aSzDel, /* Size decreases */ - int nChng /* Change in the number of documents */ + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ ){ char *pBlob; /* Storage for BLOB written into %_stat */ int nBlob; /* Size of BLOB written into %_stat */ u32 *a; /* Array of integers that becomes the BLOB */ sqlite3_stmt *pStmt; /* Statement for reading and writing */ int i; /* Loop counter */ int rc; /* Result code from subfunctions */ + const int nStat = p->nColumn+2; + if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*(p->nColumn+1) ); + a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; } - pBlob = (char*)&a[p->nColumn+1]; + pBlob = (char*)&a[nStat]; rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; } if( sqlite3_step(pStmt)==SQLITE_ROW ){ - fts3DecodeIntArray(p->nColumn+1, a, + fts3DecodeIntArray(nStat, a, sqlite3_column_blob(pStmt, 0), sqlite3_column_bytes(pStmt, 0)); }else{ - memset(a, 0, sizeof(u32)*(p->nColumn+1) ); + memset(a, 0, sizeof(u32)*(nStat) ); } sqlite3_reset(pStmt); if( nChng<0 && a[0]<(u32)(-nChng) ){ a[0] = 0; }else{ a[0] += nChng; } - for(i=0; inColumn; i++){ + for(i=0; inColumn+1; i++){ u32 x = a[i+1]; if( x+aSzIns[i] < aSzDel[i] ){ x = 0; }else{ x = x + aSzIns[i] - aSzDel[i]; } a[i+1] = x; } - fts3EncodeIntArray(p->nColumn+1, a, pBlob, &nBlob); + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); rc = fts3SqlStmt(p, SQL_REPLACE_DOCTOTAL, &pStmt, 0); if( rc ){ sqlite3_free(a); *pRC = rc; return; @@ -113148,13 +116774,163 @@ rc = SQLITE_OK; #endif }else{ rc = SQLITE_ERROR; } + + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** Return the deferred doclist associated with deferred token pDeferred. +** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already +** been called to allocate and populate the doclist. +*/ +SQLITE_PRIVATE char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ + if( pDeferred->pList ){ + *pnByte = pDeferred->pList->nData; + return pDeferred->pList->aData; + } + *pnByte = 0; + return 0; +} + +/* +** Helper fucntion for FreeDeferredDoclists(). This function removes all +** references to deferred doclists from within the tree of Fts3Expr +** structures headed by +*/ +static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ + if( pExpr ){ + fts3DeferredDoclistClear(pExpr->pLeft); + fts3DeferredDoclistClear(pExpr->pRight); + if( pExpr->isLoaded ){ + sqlite3_free(pExpr->aDoclist); + pExpr->isLoaded = 0; + pExpr->aDoclist = 0; + pExpr->nDoclist = 0; + pExpr->pCurrent = 0; + pExpr->iCurrent = 0; + } + } +} + +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + sqlite3_free(pDef->pList); + pDef->pList = 0; + } + if( pCsr->pDeferred ){ + fts3DeferredDoclistClear(pCsr->pExpr); + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + sqlite3_free(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = pModule->xOpen(pT, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken; /* Number of bytes in token */ + int iDum1, iDum2; /* Dummy variables */ + int iPos; /* Position of token in zText */ + + pTC->pTokenizer = pT; + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } return rc; } + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} + /* ** This function does the work for the xUpdate method of FTS3 virtual ** tables. */ @@ -113170,20 +116946,21 @@ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ u32 *aSzIns; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ + assert( p->pSegments==0 ); /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*p->nColumn*2 ); + aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); if( aSzIns==0 ) return SQLITE_NOMEM; - aSzDel = &aSzIns[p->nColumn]; - memset(aSzIns, 0, sizeof(aSzIns[0])*p->nColumn*2); + aSzDel = &aSzIns[p->nColumn+1]; + memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); /* If this is a DELETE or UPDATE operation, remove the old record. */ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - int isEmpty; + int isEmpty = 0; rc = fts3IsEmpty(p, apVal, &isEmpty); if( rc==SQLITE_OK ){ if( isEmpty ){ /* Deleting this row means the whole table is empty. In this case ** delete the contents of all three tables and throw away any @@ -113196,12 +116973,12 @@ rc = fts3PendingTermsDocid(p, iRemove); fts3DeleteTerms(&rc, p, apVal, aSzDel); fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, apVal); if( p->bHasDocsize ){ fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, apVal); - nChng--; } + nChng--; } } }else if( sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ sqlite3_free(aSzIns); return fts3SpecialInsert(p, apVal[p->nColumn+2]); @@ -113215,20 +116992,21 @@ } if( rc==SQLITE_OK ){ rc = fts3InsertTerms(p, apVal, aSzIns); } if( p->bHasDocsize ){ - nChng++; fts3InsertDocsize(&rc, p, aSzIns); } + nChng++; } - if( p->bHasDocsize ){ + if( p->bHasStat ){ fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } sqlite3_free(aSzIns); + sqlite3Fts3SegmentsClose(p); return rc; } /* ** Flush any data in the pending-terms hash table to disk. If successful, @@ -113248,10 +117026,11 @@ }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } + sqlite3Fts3SegmentsClose(p); return rc; } #endif @@ -113271,18 +117050,34 @@ */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + /* ** Used as an fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; struct LoadDoclistCtx { - Fts3Table *pTab; /* FTS3 Table */ + Fts3Cursor *pCsr; /* FTS3 Cursor */ int nPhrase; /* Number of phrases seen so far */ int nToken; /* Number of tokens seen so far */ }; /* @@ -113324,10 +117119,12 @@ */ typedef struct MatchInfo MatchInfo; struct MatchInfo { Fts3Cursor *pCursor; /* FTS3 Cursor */ int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ u32 *aMatchinfo; /* Pre-allocated buffer */ }; @@ -113472,11 +117269,11 @@ p->nPhrase++; p->nToken += pExpr->pPhrase->nToken; if( pExpr->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pTab, pExpr); + rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); pExpr->isLoaded = 1; if( rc==SQLITE_OK ){ rc = fts3ExprNearTrim(pExpr); } } @@ -113515,19 +117312,31 @@ int *pnPhrase, /* OUT: Number of phrases in query */ int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ - sCtx.pTab = (Fts3Table *)pCsr->base.pVtab; + sCtx.pCsr = pCsr; rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb1, (void *)&sCtx); if( rc==SQLITE_OK ){ (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0); } if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; } + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + UNUSED_PARAMETER(pExpr); + UNUSED_PARAMETER(iPhrase); + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} /* ** Advance the position list iterator specified by the first two ** arguments so that it points to the first element with a value greater ** than or equal to parameter iNext. @@ -114037,51 +117846,100 @@ *pp = pCsr; } /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. The "global" stats are those elements of the matchinfo -** array that are constant for all rows returned by the current query. +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor; + char *pIter; char *pEnd; - const int iStart = 2 + (iPhrase * p->nCol * 3) + 1; + char *pFree = 0; + u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; assert( pExpr->isLoaded ); + assert( pExpr->eType==FTSQUERY_PHRASE ); + + if( pCsr->pDeferred ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int ii; + for(ii=0; iinToken; ii++){ + if( pPhrase->aToken[ii].bFulltext ) break; + } + if( iinToken ){ + int nFree = 0; + int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); + if( rc!=SQLITE_OK ) return rc; + pIter = pFree; + pEnd = &pFree[nFree]; + }else{ + int iCol; /* Column index */ + for(iCol=0; iColnCol; iCol++){ + aOut[iCol*3 + 1] = (u32)p->nDoc; + aOut[iCol*3 + 2] = (u32)p->nDoc; + } + return SQLITE_OK; + } + }else{ + pIter = pExpr->aDoclist; + pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + } /* Fill in the global hit count matrix row for this phrase. */ - pCsr = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; - while( pCsraMatchinfo[iStart], 1); + while( pIteraDoclist ){ char *pCsr; - int iStart = 2 + (iPhrase * p->nCol * 3); + int iStart = iPhrase * p->nCol * 3; int i; for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1); @@ -114090,72 +117948,404 @@ } } return SQLITE_OK; } + +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + ){ + return SQLITE_OK; + } + *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + int nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen +){ + sqlite3_stmt *pStmt; + const char *a; + sqlite3_int64 nDoc; + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + + a = sqlite3_column_blob(pStmt, 0); + a += sqlite3Fts3GetVarint(a, &nDoc); + *pnDoc = (u32)nDoc; + + if( paLen ) *paLen = a; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPosOffset; /* Tokens count up to end of this phrase */ + int iCol; /* Current column number */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; + + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 ){ + pIter->iCol = LCS_ITERATOR_FINISHED; + rc = 1; + }else{ + if( iRead==1 ){ + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + pIter->iCol = (int)iRead; + pIter->iPos = pIter->iPosOffset; + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + rc = 1; + } + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1); + if( pIter->pRead ){ + pIter->iPos = pIter->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + }else{ + pIter->iCol = LCS_ITERATOR_FINISHED; + } + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + /* Loop through the iterators in aIter[]. Set nLive to the number of + ** iterators that point to a position-list corresponding to column iCol. + */ + for(i=0; inPhrase; i++){ + assert( aIter[i].iCol>=iCol ); + if( aIter[i].iCol==iCol ) nLive++; + } + + /* The following loop runs until all iterators in aIter[] have finished + ** iterating through positions in column iCol. Exactly one of the + ** iterators is advanced each time the body of the loop is run. + */ + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( iCol!=pIter->iCol ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + sqlite3_free(aIter); + return SQLITE_OK; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + /* ** Populate pCsr->aMatchinfo[] with data for the current row. The ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int fts3GetMatchinfo(Fts3Cursor *pCsr){ +static int fts3GetMatchinfo( + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ MatchInfo sInfo; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + memset(&sInfo, 0, sizeof(MatchInfo)); sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ + assert( pCsr->aMatchinfo ); + sqlite3_free(pCsr->aMatchinfo); + pCsr->zMatchinfo = 0; + pCsr->aMatchinfo = 0; + } + + /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ if( pCsr->aMatchinfo==0 ){ - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the - ** matchinfo function has been called for this query. In this case - ** allocate the array used to accumulate the matchinfo data and - ** initialize those elements that are constant for every row. - */ - int nPhrase; /* Number of phrases */ - int nMatchinfo; /* Number of u32 elements in match-info */ - - /* Load doclists for each phrase in the query. */ - rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - nMatchinfo = 2 + 3*sInfo.nCol*nPhrase; - if( pTab->bHasDocsize ){ - nMatchinfo += 1 + 2*pTab->nColumn; - } - - sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo); - if( !sInfo.aMatchinfo ){ - return SQLITE_NOMEM; - } - memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo); - - - /* First element of match-info is the number of phrases in the query */ - sInfo.aMatchinfo[0] = nPhrase; - sInfo.aMatchinfo[1] = sInfo.nCol; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo); - if( pTab->bHasDocsize ){ - int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; - rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]); - } - pCsr->aMatchinfo = sInfo.aMatchinfo; + int nMatchinfo = 0; /* Number of u32 elements in match-info */ + int nArg; /* Bytes in zArg */ + int i; /* Used to iterate through zArg */ + + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; + + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } + + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + nArg = (int)strlen(zArg); + pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); + if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; + + pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; + pCsr->nMatchinfo = nMatchinfo; + memcpy(pCsr->zMatchinfo, zArg, nArg+1); + memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; } sInfo.aMatchinfo = pCsr->aMatchinfo; - if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){ - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo); - if( pTab->bHasDocsize ){ - int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1]; - rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]); - } + sInfo.nPhrase = pCsr->nPhrase; + if( pCsr->isMatchinfoNeeded ){ + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); pCsr->isMatchinfoNeeded = 0; } - return SQLITE_OK; + return rc; } /* ** Implementation of snippet() function. */ @@ -114212,11 +118402,11 @@ /* Loop through all columns of the table being considered for snippets. ** If the iCol argument to this function was negative, this means all ** columns of the FTS3 table. Otherwise, only column iCol is considered. */ for(iRead=0; iReadnColumn; iRead++){ - SnippetFragment sF; + SnippetFragment sF = {0, 0, 0, 0}; int iS; if( iCol>=0 && iRead!=iCol ) continue; /* Find the best snippet of nFToken tokens in column iRead. */ rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); @@ -114246,10 +118436,11 @@ i, (i==nSnippet-1), nFToken, zStart, zEnd, zEllipsis, &res ); } snippet_out: + sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, -1, sqlite3_free); @@ -114425,10 +118616,11 @@ } offsets_out: sqlite3_free(sCtx.aTerm); assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pCtx, rc); sqlite3_free(res.z); }else{ sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); @@ -114437,25 +118629,47 @@ } /* ** Implementation of matchinfo() function. */ -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc; + int i; + const char *zFormat; + + if( zArg ){ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + return; + } + } + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + if( !pCsr->pExpr ){ sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); return; } - rc = fts3GetMatchinfo(pCsr); + + /* Retrieve matchinfo() data. */ + rc = fts3GetMatchinfo(pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ sqlite3_result_error_code(pContext, rc); }else{ - Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; - int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3); - if( pTab->bHasDocsize ){ - n += sizeof(u32)*(1 + 2*pTab->nColumn); - } + int n = pCsr->nMatchinfo * sizeof(u32); sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); } } #endif @@ -114474,10 +118688,49 @@ ** ************************************************************************* ** This file contains code for implementations of the r-tree and r*-tree ** algorithms packaged as an SQLite virtual table module. */ + +/* +** Database Format of R-Tree Tables +** -------------------------------- +** +** The data structure for a single virtual r-tree table is stored in three +** native SQLite tables declared as follows. In each case, the '%' character +** in the table name is replaced with the user-supplied name of the r-tree +** table. +** +** CREATE TABLE %_node(nodeno INTEGER PRIMARY KEY, data BLOB) +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER) +** +** The data for each node of the r-tree structure is stored in the %_node +** table. For each node that is not the root node of the r-tree, there is +** an entry in the %_parent table associating the node with its parent. +** And for each row of data in the table, there is an entry in the %_rowid +** table that maps from the entries rowid to the id of the node that it +** is stored on. +** +** The root node of an r-tree always exists, even if the r-tree table is +** empty. The nodeno of the root node is always 1. All other nodes in the +** table must be the same size as the root node. The content of each node +** is formatted as follows: +** +** 1. If the node is the root node (node 1), then the first 2 bytes +** of the node contain the tree depth as a big-endian integer. +** For non-root nodes, the first 2 bytes are left unused. +** +** 2. The next 2 bytes contain the number of entries currently +** stored in the node. +** +** 3. The remainder of the node contains the node entries. Each entry +** consists of a single 8-byte integer followed by an even number +** of 4-byte coordinates. For leaf nodes the integer is the rowid +** of a record. For internal nodes it is the node number of a +** child page. +*/ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) /* ** This file contains an implementation of a couple of different variants @@ -114515,28 +118768,40 @@ #endif #if VARIANT_RSTARTREE_SPLIT #define AssignCells splitNodeStartree #endif +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif #ifndef SQLITE_CORE SQLITE_EXTENSION_INIT1 #else #endif #ifndef SQLITE_AMALGAMATION +#include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; typedef unsigned int u32; #endif + +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif typedef struct Rtree Rtree; typedef struct RtreeCursor RtreeCursor; typedef struct RtreeNode RtreeNode; typedef struct RtreeCell RtreeCell; typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -114602,10 +118867,19 @@ */ #define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) #define RTREE_REINSERT(p) RTREE_MINCELLS(p) #define RTREE_MAXCELLS 51 +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + /* ** An rtree cursor object. */ struct RtreeCursor { sqlite3_vtab_cursor base; @@ -114634,39 +118908,27 @@ /* ** A search constraint. */ struct RtreeConstraint { - int iCoord; /* Index of constrained coordinate */ - int op; /* Constraining operation */ - double rValue; /* Constraint value. */ + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + double rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 +#define RTREE_EQ 0x41 +#define RTREE_LE 0x42 +#define RTREE_LT 0x43 +#define RTREE_GE 0x44 +#define RTREE_GT 0x45 +#define RTREE_MATCH 0x46 /* ** An rtree structure node. -** -** Data format (RtreeNode.zData): -** -** 1. If the node is the root node (node 1), then the first 2 bytes -** of the node contain the tree depth as a big-endian integer. -** For non-root nodes, the first 2 bytes are left unused. -** -** 2. The next 2 bytes contain the number of entries currently -** stored in the node. -** -** 3. The remainder of the node contains the node entries. Each entry -** consists of a single 8-byte integer followed by an even number -** of 4-byte coordinates. For leaf nodes the integer is the rowid -** of a record. For internal nodes it is the node number of a -** child page. */ struct RtreeNode { RtreeNode *pParent; /* Parent node */ i64 iNode; int nRef; @@ -114682,10 +118944,44 @@ struct RtreeCell { i64 iRowid; RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; }; + +/* +** Value for the first field of every RtreeMatchArg object. The MATCH +** operator tests that the first field of a blob operand matches this +** value to avoid operating on invalid blobs (which could cause a segfault). +*/ +#define RTREE_GEOMETRY_MAGIC 0x891245AB + +/* +** An instance of this structure must be supplied as a blob argument to +** the right-hand-side of an SQL MATCH operator used to constrain an +** r-tree query. +*/ +struct RtreeMatchArg { + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; + int nParam; + double aParam[1]; +}; + +/* +** When a geometry callback is created (see sqlite3_rtree_geometry_callback), +** a single instance of the following structure is allocated. It is used +** as the context for the user-function created by by s_r_g_c(). The object +** is eventually deleted by the destructor mechanism provided by +** sqlite3_create_function_v2() (which is called by s_r_g_c() to create +** the geometry callback function). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *); + void *pContext; +}; + #ifndef MAX # define MAX(x,y) ((x) < (y) ? (y) : (x)) #endif #ifndef MIN # define MIN(x,y) ((x) > (y) ? (y) : (x)) @@ -114764,14 +119060,12 @@ /* ** Clear the content of node p (set all bytes to 0x00). */ static void nodeZero(Rtree *pRtree, RtreeNode *p){ - if( p ){ - memset(&p->zData[2], 0, pRtree->iNodeSize-2); - p->isDirty = 1; - } + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; } /* ** Given a node number iNode, return the corresponding key to use ** in the Rtree.aHash table. @@ -114787,26 +119081,23 @@ ** Search the node hash table for node iNode. If found, return a pointer ** to it. Otherwise, return 0. */ static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ RtreeNode *p; - assert( iNode!=0 ); for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); return p; } /* ** Add node pNode to the node hash table. */ static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ - if( pNode ){ - int iHash; - assert( pNode->pNext==0 ); - iHash = nodeHash(pNode->iNode); - pNode->pNext = pRtree->aHash[iHash]; - pRtree->aHash[iHash] = pNode; - } + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; } /* ** Remove node pNode from the node hash table. */ @@ -114824,15 +119115,15 @@ ** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), ** indicating that node has not yet been assigned a node number. It is ** assigned a node number when nodeWrite() is called to write the ** node contents out to the database. */ -static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent, int zero){ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ RtreeNode *pNode; pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); if( pNode ){ - memset(pNode, 0, sizeof(RtreeNode) + (zero?pRtree->iNodeSize:0)); + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); pNode->zData = (u8 *)&pNode[1]; pNode->nRef = 1; pNode->pParent = pParent; pNode->isDirty = 1; nodeReference(pParent); @@ -114849,10 +119140,11 @@ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ RtreeNode **ppNode /* OUT: Acquired node */ ){ int rc; + int rc2 = SQLITE_OK; RtreeNode *pNode; /* Check if the requested node is already in the hash table. If so, ** increase its reference count and return it. */ @@ -114865,43 +119157,67 @@ pNode->nRef++; *ppNode = pNode; return SQLITE_OK; } - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); - if( !pNode ){ - *ppNode = 0; - return SQLITE_NOMEM; - } - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); rc = sqlite3_step(pRtree->pReadNode); if( rc==SQLITE_ROW ){ const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - assert( sqlite3_column_bytes(pRtree->pReadNode, 0)==pRtree->iNodeSize ); - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); + if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ + pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc2 = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + memcpy(pNode->zData, zBlob, pRtree->iNodeSize); + nodeReference(pParent); + } + } + } + rc = sqlite3_reset(pRtree->pReadNode); + if( rc==SQLITE_OK ) rc = rc2; + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT; + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT; + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT; + } + *ppNode = pNode; }else{ sqlite3_free(pNode); - pNode = 0; - } - - *ppNode = pNode; - rc = sqlite3_reset(pRtree->pReadNode); - - if( rc==SQLITE_OK && iNode==1 ){ - pRtree->iDepth = readInt16(pNode->zData); - } - - assert( (rc==SQLITE_OK && pNode) || (pNode==0 && rc!=SQLITE_OK) ); - nodeHashInsert(pRtree, pNode); + *ppNode = 0; + } return rc; } /* @@ -114950,12 +119266,11 @@ int nMaxCell; /* Maximum number of cells for pNode */ nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; nCell = NCELL(pNode); - assert(nCell<=nMaxCell); - + assert( nCell<=nMaxCell ); if( nCellzData[2], nCell+1); pNode->isDirty = 1; } @@ -115171,18 +119486,37 @@ *ppCursor = (sqlite3_vtab_cursor *)pCsr; return rc; } + +/* +** Free the RtreeCursor.aConstraint[] array and its contents. +*/ +static void freeCursorConstraints(RtreeCursor *pCsr){ + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; + if( pGeom ){ + if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); + sqlite3_free(pGeom); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } +} + /* ** Rtree virtual table module xClose method. */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); int rc; RtreeCursor *pCsr = (RtreeCursor *)cur; - sqlite3_free(pCsr->aConstraint); + freeCursorConstraints(pCsr); rc = nodeRelease(pRtree, pCsr->pNode); sqlite3_free(pCsr); return rc; } @@ -115194,75 +119528,134 @@ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; return (pCsr->pNode==0); } + +/* +** The r-tree constraint passed as the second argument to this function is +** guaranteed to be a MATCH constraint. +*/ +static int testRtreeGeom( + Rtree *pRtree, /* R-Tree object */ + RtreeConstraint *pConstraint, /* MATCH constraint to test */ + RtreeCell *pCell, /* Cell to test */ + int *pbRes /* OUT: Test result */ +){ + int i; + double aCoord[RTREE_MAX_DIMENSIONS*2]; + int nCoord = pRtree->nDim*2; + + assert( pConstraint->op==RTREE_MATCH ); + assert( pConstraint->pGeom ); + + for(i=0; iaCoord[i]); + } + return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); +} /* ** Cursor pCursor currently points to a cell in a non-leaf page. -** Return true if the sub-tree headed by the cell is filtered +** Set *pbEof to true if the sub-tree headed by the cell is filtered ** (excluded) by the constraints in the pCursor->aConstraint[] ** array, or false otherwise. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; int bRes = 0; + int rc = SQLITE_OK; nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; bRes==0 && iinConstraint; ii++){ RtreeConstraint *p = &pCursor->aConstraint[ii]; double cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); double cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ - case RTREE_LE: case RTREE_LT: bRes = p->rValuerValue>cell_max; break; - case RTREE_EQ: + case RTREE_LE: case RTREE_LT: + bRes = p->rValuerValue>cell_max; + break; + + case RTREE_EQ: bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &bRes); + bRes = !bRes; + break; + } } } - return bRes; + *pbEof = bRes; + return rc; } /* -** Return true if the cell that cursor pCursor currently points to +** Test if the cell that cursor pCursor currently points to ** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array, or false otherwise. +** pCursor->aConstraint[] array. If so, set *pbEof to true before +** returning. If the cell is not filtered (excluded) by the constraints, +** set pbEof to zero. +** +** Return SQLITE_OK if successful or an SQLite error code if an error +** occurs within a geometry callback. ** ** This function assumes that the cell is part of a leaf node. */ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor){ +static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ RtreeCell cell; int ii; + *pbEof = 0; nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); for(ii=0; iinConstraint; ii++){ RtreeConstraint *p = &pCursor->aConstraint[ii]; double coord = DCOORD(cell.aCoord[p->iCoord]); int res; assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH ); switch( p->op ){ case RTREE_LE: res = (coord<=p->rValue); break; case RTREE_LT: res = (coordrValue); break; case RTREE_GE: res = (coord>=p->rValue); break; case RTREE_GT: res = (coord>p->rValue); break; case RTREE_EQ: res = (coord==p->rValue); break; + default: { + int rc; + assert( p->op==RTREE_MATCH ); + rc = testRtreeGeom(pRtree, p, &cell, &res); + if( rc!=SQLITE_OK ){ + return rc; + } + break; + } } - if( !res ) return 1; + if( !res ){ + *pbEof = 1; + return SQLITE_OK; + } } - return 0; + return SQLITE_OK; } /* ** Cursor pCursor currently points at a node that heads a sub-tree of ** height iHeight (if iHeight==0, then the node is a leaf). Descend @@ -115285,33 +119678,32 @@ int iSavedCell = pCursor->iCell; assert( iHeight>=0 ); if( iHeight==0 ){ - isEof = testRtreeEntry(pRtree, pCursor); + rc = testRtreeEntry(pRtree, pCursor, &isEof); }else{ - isEof = testRtreeCell(pRtree, pCursor); + rc = testRtreeCell(pRtree, pCursor, &isEof); } - if( isEof || iHeight==0 ){ - *pEof = isEof; - return SQLITE_OK; + if( rc!=SQLITE_OK || isEof || iHeight==0 ){ + goto descend_to_cell_out; } iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); if( rc!=SQLITE_OK ){ - return rc; + goto descend_to_cell_out; } nodeRelease(pRtree, pCursor->pNode); pCursor->pNode = pChild; isEof = 1; for(ii=0; isEof && iiiCell = ii; rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); if( rc!=SQLITE_OK ){ - return rc; + goto descend_to_cell_out; } } if( isEof ){ assert( pCursor->pNode==pChild ); @@ -115319,53 +119711,68 @@ nodeRelease(pRtree, pChild); pCursor->pNode = pSavedNode; pCursor->iCell = iSavedCell; } +descend_to_cell_out: *pEof = isEof; - return SQLITE_OK; + return rc; } /* ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. */ -static int nodeRowidIndex(Rtree *pRtree, RtreeNode *pNode, i64 iRowid){ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ int ii; - for(ii=0; nodeGetRowid(pRtree, pNode, ii)!=iRowid; ii++){ - assert( ii<(NCELL(pNode)-1) ); + int nCell = NCELL(pNode); + for(ii=0; iipParent; if( pParent ){ - return nodeRowidIndex(pRtree, pParent, pNode->iNode); + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); } - return -1; + *piIndex = -1; + return SQLITE_OK; } /* ** Rtree virtual table module xNext method. */ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ Rtree *pRtree = (Rtree *)(pVtabCursor->pVtab); RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; + + /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is + ** already at EOF. It is against the rules to call the xNext() method of + ** a cursor that has already reached EOF. + */ + assert( pCsr->pNode ); if( pCsr->iStrategy==1 ){ /* This "scan" is a direct lookup by rowid. There is no next entry. */ nodeRelease(pRtree, pCsr->pNode); pCsr->pNode = 0; - } - - else if( pCsr->pNode ){ + }else{ /* Move to the next entry that matches the configured constraints. */ int iHeight = 0; while( pCsr->pNode ){ RtreeNode *pNode = pCsr->pNode; int nCell = NCELL(pNode); @@ -115375,11 +119782,14 @@ if( rc!=SQLITE_OK || !isEof ){ return rc; } } pCsr->pNode = pNode->pParent; - pCsr->iCell = nodeParentIndex(pRtree, pNode); + rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); + if( rc!=SQLITE_OK ){ + return rc; + } nodeReference(pCsr->pNode); nodeRelease(pRtree, pNode); iHeight++; } } @@ -115443,10 +119853,55 @@ rc = sqlite3_reset(pRtree->pReadRowid); } return rc; } +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *p; + sqlite3_rtree_geometry *pGeom; + int nBlob; + + /* Check that value is actually a blob. */ + if( !sqlite3_value_type(pValue)==SQLITE_BLOB ) return SQLITE_ERROR; + + /* Check that the blob is roughly the right size. */ + nBlob = sqlite3_value_bytes(pValue); + if( nBlob<(int)sizeof(RtreeMatchArg) + || ((nBlob-sizeof(RtreeMatchArg))%sizeof(double))!=0 + ){ + return SQLITE_ERROR; + } + + pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( + sizeof(sqlite3_rtree_geometry) + nBlob + ); + if( !pGeom ) return SQLITE_NOMEM; + memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); + p = (RtreeMatchArg *)&pGeom[1]; + + memcpy(p, sqlite3_value_blob(pValue), nBlob); + if( p->magic!=RTREE_GEOMETRY_MAGIC + || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(double)) + ){ + sqlite3_free(pGeom); + return SQLITE_ERROR; + } + + pGeom->pContext = p->pContext; + pGeom->nParam = p->nParam; + pGeom->aParam = p->aParam; + + pCons->xGeom = p->xGeom; + pCons->pGeom = pGeom; + return SQLITE_OK; +} /* ** Rtree virtual table module xFilter method. */ static int rtreeFilter( @@ -115461,22 +119916,22 @@ int ii; int rc = SQLITE_OK; rtreeReference(pRtree); - sqlite3_free(pCsr->aConstraint); - pCsr->aConstraint = 0; + freeCursorConstraints(pCsr); pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ i64 iRowid = sqlite3_value_int64(argv[0]); rc = findLeafNode(pRtree, iRowid, &pLeaf); pCsr->pNode = pLeaf; - if( pLeaf && rc==SQLITE_OK ){ - pCsr->iCell = nodeRowidIndex(pRtree, pLeaf, iRowid); + if( pLeaf ){ + assert( rc==SQLITE_OK ); + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ @@ -115484,16 +119939,28 @@ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); assert( (idxStr==0 && argc==0) || strlen(idxStr)==argc*2 ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; p->iCoord = idxStr[ii*2+1]-'a'; - p->rValue = sqlite3_value_double(argv[ii]); + if( p->op==RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + }else{ + p->rValue = sqlite3_value_double(argv[ii]); + } } } } if( rc==SQLITE_OK ){ @@ -115530,15 +119997,14 @@ ** least desirable): ** ** idxNum idxStr Strategy ** ------------------------------------------------ ** 1 Unused Direct lookup by rowid. -** 2 See below R-tree query. -** 3 Unused Full table scan. +** 2 See below R-tree query or full-table scan. ** ------------------------------------------------ ** -** If strategy 1 or 3 is used, then idxStr is not meaningful. If strategy +** If strategy 1 is used, then idxStr is not meaningful. If strategy ** 2 is used, idxStr is formatted to contain 2 bytes for each ** constraint used. The first two bytes of idxStr correspond to ** the constraint in sqlite3_index_info.aConstraintUsage[] with ** (argvIndex==1) etc. ** @@ -115550,10 +120016,11 @@ ** = 0x41 ('A') ** <= 0x42 ('B') ** < 0x43 ('C') ** >= 0x44 ('D') ** > 0x45 ('E') +** MATCH 0x46 ('F') ** ---------------------- ** ** The second of each pair of bytes identifies the coordinate column ** to which the constraint applies. The leftmost coordinate column ** is 'a', the second from the left 'b' etc. @@ -115563,10 +120030,11 @@ int ii, cCol; int iIdx = 0; char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; memset(zIdxStr, 0, sizeof(zIdxStr)); + UNUSED_PARAMETER(tab); assert( pIdxInfo->idxStr==0 ); for(ii=0; iinConstraint; ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; @@ -115588,43 +120056,47 @@ */ pIdxInfo->estimatedCost = 10.0; return SQLITE_OK; } - if( p->usable && p->iColumn>0 ){ + if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + int j, opmsk; + static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; u8 op = 0; switch( p->op ){ case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; - } - if( op ){ - /* Make sure this particular constraint has not been used before. - ** If it has been used before, ignore it. - ** - ** A <= or < can be used if there is a prior >= or >. - ** A >= or > can be used if there is a prior < or <=. - ** A <= or < is disqualified if there is a prior <=, <, or ==. - ** A >= or > is disqualified if there is a prior >=, >, or ==. - ** A == is disqualifed if there is any prior constraint. - */ - int j, opmsk; - static const unsigned char compatible[] = { 0, 0, 1, 1, 2, 2 }; - assert( compatible[RTREE_EQ & 7]==0 ); - assert( compatible[RTREE_LT & 7]==1 ); - assert( compatible[RTREE_LE & 7]==1 ); - assert( compatible[RTREE_GT & 7]==2 ); - assert( compatible[RTREE_GE & 7]==2 ); - cCol = p->iColumn - 1 + 'a'; - opmsk = compatible[op & 7]; - for(j=0; jop==SQLITE_INDEX_CONSTRAINT_MATCH ); + op = RTREE_MATCH; + break; + } + assert( op!=0 ); + + /* Make sure this particular constraint has not been used before. + ** If it has been used before, ignore it. + ** + ** A <= or < can be used if there is a prior >= or >. + ** A >= or > can be used if there is a prior < or <=. + ** A <= or < is disqualified if there is a prior <=, <, or ==. + ** A >= or > is disqualified if there is a prior >=, >, or ==. + ** A == is disqualifed if there is any prior constraint. + */ + assert( compatible[RTREE_EQ & 7]==0 ); + assert( compatible[RTREE_LT & 7]==1 ); + assert( compatible[RTREE_LE & 7]==1 ); + assert( compatible[RTREE_GT & 7]==2 ); + assert( compatible[RTREE_GE & 7]==2 ); + cCol = p->iColumn - 1 + 'a'; + opmsk = compatible[op & 7]; + for(j=0; jnDim*2); jj+=2){ double x1; double x2; @@ -115821,26 +120299,35 @@ /* Select the child node which will be enlarged the least if pCell ** is inserted into it. Resolve ties by choosing the entry with ** the smallest area. */ for(iCell=0; iCelliDepth-1) ){ overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); } -#endif if( (iCell==0) || (overlappParent ){ - RtreeCell cell; RtreeNode *pParent = p->pParent; - int iCell = nodeParentIndex(pRtree, p); + RtreeCell cell; + int iCell; + + if( nodeParentIndex(pRtree, p, &iCell) ){ + return SQLITE_CORRUPT; + } nodeGetCell(pRtree, pParent, iCell, &cell); if( !cellContains(pRtree, &cell, pCell) ){ cellUnion(pRtree, &cell, pCell); nodeOverwriteCell(pRtree, pParent, &cell, iCell); } p = pParent; } + return SQLITE_OK; } /* ** Write mapping (iRowid->iNode) to the _rowid table. */ @@ -116406,18 +120898,18 @@ nodeZero(pRtree, pNode); memcpy(&aCell[nCell], pCell, sizeof(RtreeCell)); nCell++; if( pNode->iNode==1 ){ - pRight = nodeNew(pRtree, pNode, 1); - pLeft = nodeNew(pRtree, pNode, 1); + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); pRtree->iDepth++; pNode->isDirty = 1; writeInt16(pNode->zData, pRtree->iDepth); }else{ pLeft = pNode; - pRight = nodeNew(pRtree, pLeft->pParent, 1); + pRight = nodeNew(pRtree, pLeft->pParent); nodeReference(pLeft); } if( !pLeft || !pRight ){ rc = SQLITE_NOMEM; @@ -116430,12 +120922,16 @@ rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } - /* Ensure both child nodes have node numbers assigned to them. */ - if( (0==pRight->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pRight))) + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) ){ goto splitnode_out; } @@ -116447,13 +120943,19 @@ if( rc!=SQLITE_OK ){ goto splitnode_out; } }else{ RtreeNode *pParent = pLeft->pParent; - int iCell = nodeParentIndex(pRtree, pLeft); - nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); - AdjustTree(pRtree, pParent, &leftbbox); + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } } if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ goto splitnode_out; } @@ -116493,44 +120995,73 @@ nodeRelease(pRtree, pLeft); sqlite3_free(aCell); return rc; } +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ int rc = SQLITE_OK; - if( pLeaf->iNode!=1 && pLeaf->pParent==0 ){ - sqlite3_bind_int64(pRtree->pReadParent, 1, pLeaf->iNode); - if( sqlite3_step(pRtree->pReadParent)==SQLITE_ROW ){ - i64 iNode = sqlite3_column_int64(pRtree->pReadParent, 0); - rc = nodeAcquire(pRtree, iNode, 0, &pLeaf->pParent); - }else{ - rc = SQLITE_ERROR; - } - sqlite3_reset(pRtree->pReadParent); - if( rc==SQLITE_OK ){ - rc = fixLeafParent(pRtree, pLeaf->pParent); - } + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( !pTest ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT; + pChild = pChild->pParent; } return rc; } static int deleteCell(Rtree *, RtreeNode *, int, int); static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ int rc; + int rc2; RtreeNode *pParent; int iCell; assert( pNode->nRef==1 ); /* Remove the entry in the parent cell. */ - iCell = nodeParentIndex(pRtree, pNode); - pParent = pNode->pParent; - pNode->pParent = 0; - if( SQLITE_OK!=(rc = deleteCell(pRtree, pParent, iCell, iHeight+1)) - || SQLITE_OK!=(rc = nodeRelease(pRtree, pParent)) - ){ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ return rc; } /* Remove the xxx_node entry. */ sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); @@ -116556,12 +121087,13 @@ pRtree->pDeleted = pNode; return SQLITE_OK; } -static void fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; if( pParent ){ int ii; int nCell = NCELL(pNode); RtreeCell box; /* Bounding box for pNode */ nodeGetCell(pRtree, pNode, 0, &box); @@ -116569,21 +121101,25 @@ RtreeCell cell; nodeGetCell(pRtree, pNode, ii, &cell); cellUnion(pRtree, &box, &cell); } box.iRowid = pNode->iNode; - ii = nodeParentIndex(pRtree, pNode); - nodeOverwriteCell(pRtree, pParent, &box, ii); - fixBoundingBox(pRtree, pParent); + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } } + return rc; } /* ** Delete the cell at index iCell of node pNode. After removing the ** cell, adjust the r-tree data structure if required. */ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; int rc; if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ return rc; } @@ -116596,18 +121132,17 @@ /* If the node is not the tree root and now has less than the minimum ** number of cells, remove it from the tree. Otherwise, update the ** cell in the parent node so that it tightly contains the updated ** node. */ - if( pNode->iNode!=1 ){ - RtreeNode *pParent = pNode->pParent; - if( (pParent->iNode!=1 || NCELL(pParent)!=1) - && (NCELL(pNode)pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)iRowid, pNode->iNode); } } } if( rc==SQLITE_OK ){ - fixBoundingBox(pRtree, pNode); + rc = fixBoundingBox(pRtree, pNode); } for(; rc==SQLITE_OK && iiiNode currently contains ** the height of the sub-tree headed by the cell. */ @@ -116740,15 +121275,17 @@ } #else rc = SplitNode(pRtree, pNode, pCell, iHeight); #endif }else{ - AdjustTree(pRtree, pNode, pCell); - if( iHeight==0 ){ - rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + rc = AdjustTree(pRtree, pNode, pCell); + if( rc==SQLITE_OK ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } } } return rc; } @@ -116789,20 +121326,10 @@ rc = sqlite3_reset(pRtree->pWriteRowid); *piRowid = sqlite3_last_insert_rowid(pRtree->db); return rc; } -#ifndef NDEBUG -static int hashIsEmpty(Rtree *pRtree){ - int ii; - for(ii=0; iiaHash[ii] ); - } - return 1; -} -#endif - /* ** The xUpdate method for rtree module virtual tables. */ static int rtreeUpdate( sqlite3_vtab *pVtab, @@ -116814,11 +121341,10 @@ int rc = SQLITE_OK; rtreeReference(pRtree); assert(nData>=1); - assert(hashIsEmpty(pRtree)); /* If azData[0] is not an SQL NULL value, it is the rowid of a ** record to delete from the r-tree table. The following block does ** just that. */ @@ -116840,12 +121366,14 @@ } /* Delete the cell in question from the leaf node. */ if( rc==SQLITE_OK ){ int rc2; - iCell = nodeRowidIndex(pRtree, pLeaf, iDelete); - rc = deleteCell(pRtree, pLeaf, iCell, 0); + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ rc = rc2; } } @@ -116863,23 +121391,24 @@ ** ** This is equivalent to copying the contents of the child into ** the root node (the operation that Gutman's paper says to perform ** in this scenario). */ - if( rc==SQLITE_OK && pRtree->iDepth>0 ){ - if( rc==SQLITE_OK && NCELL(pRoot)==1 ){ - RtreeNode *pChild; - i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); - if( rc==SQLITE_OK ){ - rc = removeNode(pRtree, pChild, pRtree->iDepth-1); - } - if( rc==SQLITE_OK ){ - pRtree->iDepth--; - writeInt16(pRoot->zData, pRtree->iDepth); - pRoot->isDirty = 1; - } + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; } } /* Re-insert the contents of any underfull nodes removed from the tree. */ for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ @@ -117165,11 +121694,11 @@ ){ int rc = SQLITE_OK; Rtree *pRtree; int nDb; /* Length of string argv[1] */ int nName; /* Length of string argv[2] */ - int eCoordType = (int)pAux; + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); const char *aErrMsg[] = { 0, /* 0 */ "Wrong number of columns for an rtree table", /* 1 */ "Too few columns for an rtree table", /* 2 */ @@ -117262,10 +121791,11 @@ char *zText = 0; RtreeNode node; Rtree tree; int ii; + UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); tree.nDim = sqlite3_value_int(apArg[0]); tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); @@ -117295,10 +121825,11 @@ sqlite3_result_text(ctx, zText, -1, sqlite3_free); } static void rtreedepth(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + UNUSED_PARAMETER(nArg); if( sqlite3_value_type(apArg[0])!=SQLITE_BLOB || sqlite3_value_bytes(apArg[0])<2 ){ sqlite3_result_error(ctx, "Invalid argument to rtreedepth()", -1); }else{ @@ -117311,18 +121842,15 @@ ** Register the r-tree module with database handle db. This creates the ** virtual table module "rtree" and the debugging/analysis scalar ** function "rtreenode". */ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db){ - int rc = SQLITE_OK; - - if( rc==SQLITE_OK ){ - int utf8 = SQLITE_UTF8; - rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); - } - if( rc==SQLITE_OK ){ - int utf8 = SQLITE_UTF8; + const int utf8 = SQLITE_UTF8; + int rc; + + rc = sqlite3_create_function(db, "rtreenode", 2, utf8, 0, rtreenode, 0, 0); + if( rc==SQLITE_OK ){ rc = sqlite3_create_function(db, "rtreedepth", 1, utf8, 0,rtreedepth, 0, 0); } if( rc==SQLITE_OK ){ void *c = (void *)RTREE_COORD_REAL32; rc = sqlite3_create_module_v2(db, "rtree", &rtreeModule, c, 0); @@ -117332,10 +121860,74 @@ rc = sqlite3_create_module_v2(db, "rtree_i32", &rtreeModule, c, 0); } return rc; } + +/* +** A version of sqlite3_free() that can be used as a callback. This is used +** in two places - as the destructor for the blob value returned by the +** invocation of a geometry function, and as the destructor for the geometry +** functions themselves. +*/ +static void doSqlite3Free(void *p){ + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite +** scalar user function. This C function is the callback used for all such +** registered SQL functions. +** +** The scalar user functions return a blob that is interpreted by r-tree +** table MATCH operators. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + int nBlob; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(double); + pBlob = (RtreeMatchArg *)sqlite3_malloc(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->magic = RTREE_GEOMETRY_MAGIC; + pBlob->xGeom = pGeomCtx->xGeom; + pBlob->pContext = pGeomCtx->pContext; + pBlob->nParam = nArg; + for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); + } + sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int, double *, int *), + void *pContext +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->pContext = pContext; + + /* Create the new user-function. Register a destructor function to delete + ** the context object when it is no longer required. */ + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + ); +} #if !SQLITE_CORE SQLITE_API int sqlite3_extension_init( sqlite3 *db, char **pzErrMsg, Index: src/sqlite3.h ================================================================== --- src/sqlite3.h +++ src/sqlite3.h @@ -95,23 +95,23 @@ ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** ** Since version 3.6.18, SQLite source code has been stored in the ** Fossil configuration management -** system. ^The SQLITE_SOURCE_ID macro evalutes to +** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID ** string contains the date and time of the check-in (UTC) and an SHA1 ** hash of the entire source tree. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.7.0" -#define SQLITE_VERSION_NUMBER 3007000 -#define SQLITE_SOURCE_ID "2010-07-09 12:57:54 0c32c4bbdd74297767dcf4ec4295f9cc72875af0" +#define SQLITE_VERSION "3.7.4" +#define SQLITE_VERSION_NUMBER 3007004 +#define SQLITE_SOURCE_ID "2010-12-06 21:09:59 fabcb6b95e1d4059d1e6c6183f65846f6cbd5749" /* ** CAPI3REF: Run-Time Library Version Numbers ** KEYWORDS: sqlite3_version, sqlite3_sourceid ** @@ -152,19 +152,19 @@ ** ^The sqlite3_compileoption_used() function returns 0 or 1 ** indicating whether the specified option was defined at ** compile time. ^The SQLITE_ prefix may be omitted from the ** option name passed to sqlite3_compileoption_used(). ** -** ^The sqlite3_compileoption_get() function allows interating +** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, ** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ ** prefix is omitted from any strings returned by ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifing the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and ** [sqlite_compileoption_get()] and the [compile_options pragma]. */ @@ -266,11 +266,11 @@ /* ** CAPI3REF: Closing A Database Connection ** ** ^The sqlite3_close() routine is the destructor for the [sqlite3] object. ** ^Calls to sqlite3_close() return SQLITE_OK if the [sqlite3] object is -** successfullly destroyed and all associated resources are deallocated. +** successfully destroyed and all associated resources are deallocated. ** ** Applications must [sqlite3_finalize | finalize] all [prepared statements] ** and [sqlite3_blob_close | close] all [BLOB handles] associated with ** the [sqlite3] object prior to attempting to close the object. ^If ** sqlite3_close() is called on a [database connection] that still has @@ -479,15 +479,16 @@ #define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ #define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ /* ** CAPI3REF: Device Characteristics ** -** The xDeviceCapabilities method of the [sqlite3_io_methods] +** The xDeviceCharacteristics method of the [sqlite3_io_methods] ** object returns an integer which is a vector of the these ** bit values expressing I/O characteristics of the mass storage ** device that holds the file that the [sqlite3_io_methods] ** refers to. ** @@ -539,10 +540,22 @@ ** sync operation only needs to flush data to mass storage. Inode ** information need not be flushed. If the lower four bits of the flag ** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. ** If the lower four bits equal SQLITE_SYNC_FULL, that means ** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) */ #define SQLITE_SYNC_NORMAL 0x00002 #define SQLITE_SYNC_FULL 0x00003 #define SQLITE_SYNC_DATAONLY 0x00010 @@ -663,15 +676,14 @@ int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); int (*xFileControl)(sqlite3_file*, int op, void *pArg); int (*xSectorSize)(sqlite3_file*); int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ - int (*xShmOpen)(sqlite3_file*); + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); - int (*xShmMap)(sqlite3_file*, int iPage, int pgsz, int, void volatile**); void (*xShmBarrier)(sqlite3_file*); - int (*xShmClose)(sqlite3_file*, int deleteFlag); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ /* Additional methods may be added in future releases */ }; /* @@ -693,16 +705,27 @@ ** layer a hint of how large the database file will grow to be during the ** current transaction. This hint is not guaranteed to be accurate but it ** is often close. The underlying VFS might choose to preallocate database ** file space based on this hint in order to help writes to the database ** file run faster. +** +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 #define SQLITE_SET_LOCKPROXYFILE 3 #define SQLITE_LAST_ERRNO 4 #define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 + /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an @@ -746,19 +769,23 @@ ** object once the object has been registered. ** ** The zName field holds the name of the VFS module. The name must ** be unique across all VFS modules. ** -** SQLite will guarantee that the zFilename parameter to xOpen +** ^SQLite guarantees that the zFilename parameter to xOpen ** is either a NULL pointer or string obtained -** from xFullPathname(). SQLite further guarantees that +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 10 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that ** the string will be valid and unchanged until xClose() is ** called. Because of the previous sentence, ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. -** If the zFilename parameter is xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. Whenever the +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** ** The flags argument to xOpen() includes all bits set in ** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] @@ -765,11 +792,11 @@ ** or [sqlite3_open16()] is used, then flags includes at least ** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** -** SQLite will also add one of the following flags to the xOpen() +** ^(SQLite will also add one of the following flags to the xOpen() ** call, depending on the object being opened: ** **
      **
    • [SQLITE_OPEN_MAIN_DB] **
    • [SQLITE_OPEN_MAIN_JOURNAL] @@ -776,11 +803,12 @@ **
    • [SQLITE_OPEN_TEMP_DB] **
    • [SQLITE_OPEN_TEMP_JOURNAL] **
    • [SQLITE_OPEN_TRANSIENT_DB] **
    • [SQLITE_OPEN_SUBJOURNAL] **
    • [SQLITE_OPEN_MASTER_JOURNAL] -**
    +**
  • [SQLITE_OPEN_WAL] +** )^ ** ** The file I/O implementation can use the object type flags to ** change the way it deals with files. For example, an application ** that does not care about crash recovery or rollback might make ** the open of a journal file a no-op. Writes to this journal would @@ -795,39 +823,40 @@ **
  • [SQLITE_OPEN_DELETEONCLOSE] **
  • [SQLITE_OPEN_EXCLUSIVE] ** ** ** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be -** deleted when it is closed. The [SQLITE_OPEN_DELETEONCLOSE] -** will be set for TEMP databases, journals and for subjournals. +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. ** -** The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the ** SQLITE_OPEN_CREATE, is used to indicate that file should always ** be created, and that it is an error if it already exists. ** It is not used to indicate the file should be opened ** for exclusive access. ** -** At least szOsFile bytes of memory are allocated by SQLite +** ^At least szOsFile bytes of memory are allocated by SQLite ** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that ** the xOpen method must set the sqlite3_file.pMethods to either ** a valid [sqlite3_io_methods] object or to NULL. xOpen must do ** this even if the open fails. SQLite expects that the sqlite3_file.pMethods ** element will be valid after xOpen returns regardless of the success ** or failure of the xOpen call. ** -** The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] ** to test whether a file is at least readable. The file can be a ** directory. ** -** SQLite will always allocate at least mxPathname+1 bytes for the +** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer ** is also passed as a parameter to both methods. If the output buffer ** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is ** handled as a fatal error by SQLite, vfs implementations should endeavor ** to prevent this by setting mxPathname to a sufficiently large value. @@ -837,14 +866,14 @@ ** included in the VFS structure for completeness. ** The xRandomness() function attempts to return nBytes bytes ** of good-quality randomness into zOut. The return value is ** the actual number of bytes of randomness obtained. ** The xSleep() method causes the calling thread to sleep for at -** least the number of microseconds given. The xCurrentTime() +** least the number of microseconds given. ^The xCurrentTime() ** method returns a Julian Day Number for the current date and time as ** a floating point value. -** The xCurrentTimeInt64() method returns, as an integer, the Julian +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian ** Day Number multipled by 86400000 (the number of milliseconds in ** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current ** date and time if that method is available (if iVersion is 2 or ** greater and the function pointer is not NULL) and will fall back @@ -890,17 +919,24 @@ ** the xAccess method of an [sqlite3_vfs] object. They determine ** what kind of permissions the xAccess method is looking for. ** With SQLITE_ACCESS_EXISTS, the xAccess method ** simply checks whether the file exists. ** With SQLITE_ACCESS_READWRITE, the xAccess method -** checks whether the file is both readable and writable. +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. ** With SQLITE_ACCESS_READ, the xAccess method -** checks whether the file is readable. +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. */ #define SQLITE_ACCESS_EXISTS 0 -#define SQLITE_ACCESS_READWRITE 1 -#define SQLITE_ACCESS_READ 2 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ /* ** CAPI3REF: Flags for the xShmLock VFS method ** ** These integer constants define the various locking operations @@ -1230,11 +1266,11 @@ ** statistics. ^(When memory allocation statistics are disabled, the ** following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] -**
    • [sqlite3_soft_heap_limit()] +**
    • [sqlite3_soft_heap_limit64()] **
    • [sqlite3_status()] **
    )^ ** ^Memory allocation statistics are enabled by default unless SQLite is ** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory ** allocation statistics are disabled by default. @@ -1244,19 +1280,18 @@ **
    ^This option specifies a static memory buffer that SQLite can use for ** scratch memory. There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scrach allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N). The sz -** argument must be a multiple of 16. The sz parameter should be a few bytes -** larger than the actual scratch space required due to internal overhead. +** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than one scratch buffer per thread. So -** N should be set to the expected maximum number of threads. ^SQLite will -** never require a scratch buffer that is more than 6 times the database -** page size. ^If SQLite needs needs additional scratch memory beyond -** what is provided by this configuration option, then +** ^SQLite will use no more than two scratch buffers per thread. So +** N should be set to twice the expected maximum number of threads. +** ^SQLite will never require a scratch buffer that is more than 6 +** times the database page size. ^If SQLite needs needs additional +** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
    ** **
    SQLITE_CONFIG_PAGECACHE
    **
    ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implemenation. @@ -1272,12 +1307,11 @@ ** argument should point to an allocation of at least sz*N bytes of memory. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then ** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** ^The implementation might use one or more of the N buffers to hold -** memory accounting information. The pointer in the first argument must +** The pointer in the first argument must ** be aligned to an 8-byte boundary or subsequent behavior of SQLite ** will be undefined.
    ** **
    SQLITE_CONFIG_HEAP
    **
    ^This option specifies a static memory buffer that SQLite will use @@ -1402,12 +1436,18 @@ ** size of each lookaside buffer slot. ^The third argument is the number of ** slots. The size of the buffer in the first argument must be greater than ** or equal to the product of the second and third arguments. The buffer ** must be aligned to an 8-byte boundary. ^If the second argument to ** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally -** rounded down to the next smaller -** multiple of 8. See also: [SQLITE_CONFIG_LOOKASIDE]
    +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
  • ** ** */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ @@ -1707,10 +1747,13 @@ */ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. ** ** Definition: A result table is memory data structure created by the ** [sqlite3_get_table()] interface. A result table records the ** complete query results from one or more queries. ** @@ -1728,11 +1771,11 @@ ** ** A result table might consist of one or more memory allocations. ** It is not safe to pass a result table directly to [sqlite3_free()]. ** A result table should be deallocated using [sqlite3_free_table()]. ** -** As an example of the result table format, suppose a query result +** ^(As an example of the result table format, suppose a query result ** is as follows: ** **
     **        Name        | Age
     **        -----------------------
    @@ -1752,31 +1795,31 @@
     **        azResult[3] = "43";
     **        azResult[4] = "Bob";
     **        azResult[5] = "28";
     **        azResult[6] = "Cindy";
     **        azResult[7] = "21";
    -** 
    +** )^ ** ** ^The sqlite3_get_table() function evaluates one or more ** semicolon-separated SQL statements in the zero-terminated UTF-8 ** string of its 2nd parameter and returns a result table to the ** pointer given in its 3rd parameter. ** ** After the application has finished with the result from sqlite3_get_table(), -** it should pass the result table pointer to sqlite3_free_table() in order to +** it must pass the result table pointer to sqlite3_free_table() in order to ** release the memory that was malloced. Because of the way the ** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling ** function must not try to call [sqlite3_free()] directly. Only ** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** ^(The sqlite3_get_table() interface is implemented as a wrapper around +** The sqlite3_get_table() interface is implemented as a wrapper around ** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access ** to any internal data structures of SQLite. It uses only the public ** interface defined here. As a consequence, errors that occur in the ** wrapper layer outside of the internal [sqlite3_exec()] call are not ** reflected in subsequent calls to [sqlite3_errcode()] or -** [sqlite3_errmsg()].)^ +** [sqlite3_errmsg()]. */ SQLITE_API int sqlite3_get_table( sqlite3 *db, /* An open database */ const char *zSql, /* SQL to be evaluated */ char ***pazResult, /* Results of the query */ @@ -1924,11 +1967,13 @@ ** by sqlite3_realloc() and the prior allocation is freed. ** ^If sqlite3_realloc() returns NULL, then the prior allocation ** is not freed. ** ** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() -** is always aligned to at least an 8 byte boundary. +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. ** ** In SQLite version 3.5.0 and 3.5.1, it was possible to define ** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in ** implementation of these routines to be omitted. That capability ** is no longer provided. Only built-in memory allocators can be used. @@ -2167,30 +2212,47 @@ ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time -** of how long that statement took to run. +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. The +** sqlite3_profile() function is considered experimental and is +** subject to change in future versions of SQLite. */ SQLITE_API void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); SQLITE_API SQLITE_EXPERIMENTAL void *sqlite3_profile(sqlite3*, void(*xProfile)(void*,const char*,sqlite3_uint64), void*); /* ** CAPI3REF: Query Progress Callbacks ** -** ^This routine configures a callback function - the -** progress callback - that is invoked periodically during long -** running calls to [sqlite3_exec()], [sqlite3_step()] and -** [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. ** ** ^If the progress callback returns non-zero, the operation is ** interrupted. This feature can be used to implement a ** "Cancel" button on a GUI progress dialog box. ** -** The progress handler must not do anything that will modify +** The progress handler callback must not do anything that will modify ** the database connection that invoked the progress handler. ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their ** database connections for the meaning of "modify" in this paragraph. ** */ @@ -2245,11 +2307,11 @@ ** ** ** If the 3rd parameter to sqlite3_open_v2() is not one of the ** combinations shown above or one of the combinations shown above combined ** with the [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], -** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_SHAREDCACHE] flags, +** [SQLITE_OPEN_SHAREDCACHE] and/or [SQLITE_OPEN_PRIVATECACHE] flags, ** then the behavior is undefined. ** ** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection ** opens in the multi-thread [threading mode] as long as the single-thread ** mode has not been set at compile-time or start-time. ^If the @@ -2370,21 +2432,26 @@ ** ^(This interface allows the size of various constructs to be limited ** on a connection by connection basis. The first parameter is the ** [database connection] whose limit is to be set or queried. The ** second parameter is one of the [limit categories] that define a ** class of constructs to be size limited. The third parameter is the -** new limit for that construct. The function returns the old limit.)^ +** new limit for that construct.)^ ** ** ^If the new limit is a negative number, the limit is unchanged. -** ^(For the limit category of SQLITE_LIMIT_XYZ there is a +** ^(For each limit category SQLITE_LIMIT_NAME there is a ** [limits | hard upper bound] -** set by a compile-time C preprocessor macro named -** [limits | SQLITE_MAX_XYZ]. +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. ** (The "_LIMIT_" in the name is changed to "_MAX_".))^ ** ^Attempts to increase a limit above its hard upper bound are ** silently truncated to the hard upper bound. ** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** ** Run-time limits are intended for use in applications that manage ** both their own internal database and also databases that are controlled ** by untrusted external sources. An example application might be a ** web browser that has its own databases for storing history and ** separate databases controlled by JavaScript applications downloaded @@ -2409,11 +2476,11 @@ ** The synopsis of the meanings of the various limits is shown below. ** Additional information is available at [limits | Limits in SQLite]. ** **
    ** ^(
    SQLITE_LIMIT_LENGTH
    -**
    The maximum size of any string or BLOB or table row.
    )^ +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_SQL_LENGTH
    **
    The maximum length of an SQL statement, in bytes.
    )^ ** ** ^(
    SQLITE_LIMIT_COLUMN
    @@ -2427,11 +2494,13 @@ ** ^(
    SQLITE_LIMIT_COMPOUND_SELECT
    **
    The maximum number of terms in a compound SELECT statement.
    )^ ** ** ^(
    SQLITE_LIMIT_VDBE_OP
    **
    The maximum number of instructions in a virtual machine program -** used to implement an SQL statement.
    )^ +** used to implement an SQL statement. This limit is not currently +** enforced, though that might be added in some future release of +** SQLite.)^ ** ** ^(
    SQLITE_LIMIT_FUNCTION_ARG
    **
    The maximum number of arguments on a function.
    )^ ** ** ^(
    SQLITE_LIMIT_ATTACHED
    @@ -2440,12 +2509,11 @@ ** ^(
    SQLITE_LIMIT_LIKE_PATTERN_LENGTH
    **
    The maximum length of the pattern argument to the [LIKE] or ** [GLOB] operators.
    )^ ** ** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    -**
    The maximum number of variables in an SQL statement that can -** be bound.
    )^ +**
    The maximum index number of any [parameter] in an SQL statement.)^ ** ** ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ **
    */ @@ -2513,16 +2581,11 @@ ** **
      **
    1. ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. ^If the schema has changed in -** a way that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text -** of the parsing error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. **
    2. ** **
    3. ** ^When an error occurs, [sqlite3_step()] will return one of the detailed ** [error codes] or [extended error codes]. ^The legacy behavior was that @@ -2531,15 +2594,20 @@ ** in order to find the underlying cause of the problem. With the "v2" prepare ** interfaces, the underlying reason for the error is returned immediately. **
    4. ** **
    5. -** ^If the value of a [parameter | host parameter] in the WHERE clause might -** change the query plan for a statement, then the statement may be -** automatically recompiled (as if there had been a schema change) on the first -** [sqlite3_step()] call following any change to the -** [sqlite3_bind_text | bindings] of the [parameter]. +** ^If the specific value bound to [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled. +** the **
    6. **
    */ SQLITE_API int sqlite3_prepare( sqlite3 *db, /* Database handle */ @@ -2577,10 +2645,24 @@ ** SQL text used to create a [prepared statement] if that statement was ** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()]. */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** the [prepared statement] X is [SELECT] statement and false (zero) if +** X is an [INSERT], [UPDATE], [DELETE], CREATE, DROP, [ANALYZE], +** [ALTER], or [REINDEX] statement. +** If X is a NULL pointer or any other kind of statement, including but +** not limited to [ATTACH], [DETACH], [COMMIT], [ROLLBACK], [RELEASE], +** [SAVEPOINT], [PRAGMA], or [VACUUM] the result of sqlite3_stmt_readonly(X) is +** undefined. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Dynamically Typed Value Object ** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} ** ** SQLite uses the sqlite3_value object to represent all values @@ -2602,11 +2684,11 @@ ** or if SQLite is run in one of reduced mutex modes ** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] ** then there is no distinction between protected and unprotected ** sqlite3_value objects and they can be used interchangeably. However, ** for maximum code portability it is recommended that applications -** still make the distinction between between protected and unprotected +** still make the distinction between protected and unprotected ** sqlite3_value objects even when not strictly required. ** ** ^The sqlite3_value objects that are passed as parameters into the ** implementation of [application-defined SQL functions] are protected. ** ^The sqlite3_value object returned by @@ -2648,11 +2730,11 @@ **
  • @VVV **
  • $VVV ** ** ** In the templates above, NNN represents an integer literal, -** and VVV represents an alphanumeric identifer.)^ ^The values of these +** and VVV represents an alphanumeric identifier.)^ ^The values of these ** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** ** ^The first argument to the sqlite3_bind_*() routines is always ** a pointer to the [sqlite3_stmt] object returned from @@ -2676,11 +2758,14 @@ ** ^If the fourth parameter is negative, the length of the string is ** the number of bytes up to the first zero terminator. ** ** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and ** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or -** string after SQLite has finished with it. ^If the fifth argument is +** string after SQLite has finished with it. ^The destructor is called +** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), +** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. ** ^If the fifth argument has the value [SQLITE_TRANSIENT], then ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. @@ -2797,10 +2882,12 @@ ** CAPI3REF: Number Of Columns In A Result Set ** ** ^Return the number of columns in the result set returned by the ** [prepared statement]. ^This routine returns 0 if pStmt is an SQL ** statement that does not return data (for example an [UPDATE]). +** +** See also: [sqlite3_data_count()] */ SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set @@ -2987,12 +3074,18 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); /* ** CAPI3REF: Number of columns in a result set ** -** ^The sqlite3_data_count(P) the number of columns in the -** of the result set of [prepared statement] P. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** +** See also: [sqlite3_column_count()] */ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes @@ -3068,22 +3161,30 @@ ** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts ** the string to UTF-8 and then returns the number of bytes. ** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** ^The value returned does not include the zero terminator at the end -** of the string. ^For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** ** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), ** even empty strings, are always zero terminated. ^The return -** value from sqlite3_column_blob() for a zero-length BLOB is an arbitrary -** pointer, possibly even a NULL pointer. -** -** ^The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 in native byte order instead of UTF-8. -** ^The zero terminator is not included in this count. +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** ** ^The object returned by [sqlite3_column_value()] is an ** [unprotected sqlite3_value] object. An unprotected sqlite3_value object ** may only be used with [sqlite3_bind_value()] and [sqlite3_result_value()]. ** If the [unprotected sqlite3_value] object returned by @@ -3124,14 +3225,14 @@ ** and atof(). SQLite does not really use these functions. It has its ** own equivalent internal routines. The atoi() and atof() names are ** used in the table for brevity and because they are familiar to most ** C programmers. ** -** ^Note that when type conversions occur, pointers returned by prior +** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or ** sqlite3_column_text16() may be invalidated. -** ^(Type conversions and pointer invalidations might occur +** Type conversions and pointer invalidations might occur ** in the following cases: ** ** )^ +** ** ** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds +** that the prior pointer references will have been modified. Other kinds ** of conversion are done in place when it is possible, but sometimes they ** are not possible and in those cases prior pointers are invalidated. ** -** ^(The safest and easiest to remember policy is to invoke these routines +** The safest and easiest to remember policy is to invoke these routines ** in one of the following ways: ** ** )^ +** ** ** In other words, you should call sqlite3_column_text(), ** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result ** into the desired format, then invoke sqlite3_column_bytes() or ** sqlite3_column_bytes16() to find the size of the result. Do not mix calls @@ -3193,21 +3294,30 @@ /* ** CAPI3REF: Destroy A Prepared Statement Object ** ** ^The sqlite3_finalize() function is called to delete a [prepared statement]. -** ^If the statement was executed successfully or not executed at all, then -** SQLITE_OK is returned. ^If execution of the statement failed then an -** [error code] or [extended error code] is returned. +** ^If the most recent evaluation of the statement encountered no errors or +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** ^This routine can be called at any point during the execution of the -** [prepared statement]. ^If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an [sqlite3_interrupt | interrupt]. -** ^Incomplete updates may be rolled back and transactions canceled, -** depending on the circumstances, and the -** [error code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object @@ -3239,40 +3349,42 @@ ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} ** KEYWORDS: {application-defined SQL function} ** KEYWORDS: {application-defined SQL functions} ** -** ^These two functions (collectively known as "function creation routines") +** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only difference between the -** two is that the second parameter, the name of the (scalar) function or -** aggregate, is encoded in UTF-8 for sqlite3_create_function() and UTF-16 -** for sqlite3_create_function16(). +** of existing SQL functions or aggregates. The only differences between +** these routines are the text encoding expected for +** the the second parameter (the name of the function being created) +** and the presence or absence of a destructor callback for +** the application data pointer. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database ** connection then application-defined SQL functions must be added ** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created or -** redefined. ^The length of the name is limited to 255 bytes, exclusive of -** the zero-terminator. Note that the name length limit is in bytes, not -** characters. ^Any attempt to create a function with a longer name -** will result in [SQLITE_ERROR] being returned. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** ** ^The third parameter (nArg) ** is the number of arguments that the SQL function or ** aggregate takes. ^If this parameter is -1, then the SQL function or ** aggregate may take any number of arguments between 0 and the limit ** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third ** parameter is less than -1 or greater than 127 then the behavior is ** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be +** its parameters. Every SQL function implementation must be able to work +** with UTF-8, UTF-16le, or UTF-16be. But some implementations may be ** more efficient with one encoding than another. ^An application may ** invoke sqlite3_create_function() or sqlite3_create_function16() multiple ** times with the same function but with different values of eTextRep. ** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. @@ -3280,17 +3392,28 @@ ** encoding is used, then the fourth argument should be [SQLITE_ANY]. ** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are +** ^The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc -** callback only; NULL pointers should be passed as the xStep and xFinal +** callback only; NULL pointers must be passed as the xStep and xFinal ** parameters. ^An aggregate SQL function requires an implementation of xStep -** and xFinal and NULL should be passed for xFunc. ^To delete an existing -** SQL function or aggregate, pass NULL for all three function callbacks. +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL poiners for all three function +** callbacks. +** +** ^(If the tenth parameter to sqlite3_create_function_v2() is not NULL, +** then it is destructor for the application data pointer. +** The destructor is invoked when the function is deleted, either by being +** overloaded or when the database connection closes.)^ +** ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. +** ^When the destructor callback of the tenth parameter is invoked, it +** is passed a single argument which is a copy of the application data +** pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of ** arguments or differing preferred text encodings. ^SQLite will use ** the implementation that most closely matches the way in which the @@ -3302,15 +3425,10 @@ ** ^A function where the encoding difference is between UTF16le and UTF16be ** is a closer match than a function where the encoding difference is ** between UTF8 and UTF16. ** ** ^Built-in functions may be overloaded by new application-defined functions. -** ^The first application-defined function with a given name overrides all -** built-in functions in the same [database connection] with the same name. -** ^Subsequent application-defined functions of the same name only override -** prior application-defined functions that are an exact match for the -** number of parameters and preferred encoding. ** ** ^An application-defined function is permitted to call other ** SQLite interfaces. However, such calls must not ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. @@ -3332,10 +3450,21 @@ int eTextRep, void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) ); /* ** CAPI3REF: Text Encodings ** @@ -3427,11 +3556,11 @@ SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context ** -** Implementions of aggregate SQL functions use this +** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called ** for a particular aggregate function, SQLite ** allocates N of memory, zeroes out that memory, and returns a pointer @@ -3679,73 +3808,106 @@ SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); /* ** CAPI3REF: Define New Collating Sequences ** -** These functions are used to add new collation sequences to the -** [database connection] specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** ^The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). ^In all cases -** the name is passed as the second function argument. -** -** ^The third argument may be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE], or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian, or UTF-16 big-endian, respectively. ^The -** third argument might also be [SQLITE_UTF16] to indicate that the routine -** expects pointers to be UTF-16 strings in the native byte order, or the -** argument can be [SQLITE_UTF16_ALIGNED] if the -** the routine expects pointers to 16-bit word aligned strings -** of UTF-16 in the native byte order. -** -** A pointer to the user supplied routine must be passed as the fifth -** argument. ^If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). -** ^Each time the application supplied function is invoked, it is passed -** as its first parameter a copy of the void* passed as the fourth argument -** to sqlite3_create_collation() or sqlite3_create_collation16(). -** -** ^The remaining arguments to the application-supplied routine are two strings, -** each represented by a (length, data) pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The application defined collation routine should -** return negative, zero or positive if the first string is less than, -** equal to, or greater than the second string. i.e. (STRING1 - STRING2). +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +** )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCallback. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is a application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCallback argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The collating function must return an +** integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must alway return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** except that it takes an extra argument which is a destructor for -** the collation. ^The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). -** ^Collations are destroyed when they are overridden by later calls to the -** collation creation functions or when the [database connection] is closed -** using [sqlite3_close()]. +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ SQLITE_API int sqlite3_create_collation( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); SQLITE_API int sqlite3_create_collation_v2( sqlite3*, const char *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); SQLITE_API int sqlite3_create_collation16( sqlite3*, const void *zName, int eTextRep, - void*, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); /* ** CAPI3REF: Collation Needed Callbacks @@ -3830,20 +3992,23 @@ #endif /* ** CAPI3REF: Suspend Execution For A Short Time ** -** ^The sqlite3_sleep() function causes the current thread to suspend execution +** The sqlite3_sleep() function causes the current thread to suspend execution ** for at least a number of milliseconds specified in its parameter. ** -** ^If the operating system does not support sleep requests with +** If the operating system does not support sleep requests with ** millisecond time resolution, then the time will be rounded up to -** the nearest second. ^The number of milliseconds of sleep actually +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. ** ** ^SQLite implements this interface by calling the xSleep() -** method of the default [sqlite3_vfs] object. +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. */ SQLITE_API int sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -4061,44 +4226,77 @@ ** of heap memory by deallocating non-essential memory allocations ** held by the database library. Memory used to cache database ** pages to improve performance is an example of non-essential memory. ** ^sqlite3_release_memory() returns the number of bytes actually freed, ** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. */ SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Impose A Limit On Heap Size ** -** ^The sqlite3_soft_heap_limit() interface places a "soft" limit -** on the amount of heap memory that may be allocated by SQLite. -** ^If an internal allocation is requested that would exceed the -** soft heap limit, [sqlite3_release_memory()] is invoked one or -** more times to free up some space before the allocation is performed. -** -** ^The limit is called "soft" because if [sqlite3_release_memory()] -** cannot free sufficient memory to prevent the limit from being exceeded, -** the memory is allocated anyway and the current operation proceeds. -** -** ^A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhausted. -** ^The default value for the soft heap limit is zero. -** -** ^(SQLite makes a best effort to honor the soft heap limit. -** But if the soft heap limit cannot be honored, execution will -** continue without error or notification.)^ This is why the limit is -** called a "soft" limit. It is advisory only. -** -** Prior to SQLite version 3.5.0, this routine only constrained the memory -** allocated by a single thread - the same thread in which this routine -** runs. Beginning with SQLite version 3.5.0, the soft heap limit is -** applied to all threads. The value specified for the soft heap limit -** is an upper bound on the total memory allocation for all threads. In -** version 3.5.0 there is no mechanism for limiting the heap usage for -** individual threads. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The return value from sqlite3_soft_heap_limit64() is the size of +** the soft heap limit prior to the call. ^If the argument N is negative +** then no change is made to the soft heap limit. Hence, the current +** size of the soft heap limit can be determined by invoking +** sqlite3_soft_heap_limit64() with a negative argument. +** +** ^If the argument N is zero then the soft heap limit is disabled. +** +** ^(The soft heap limit is not enforced in the current implementation +** if one or more of following conditions are true: +** +** )^ +** +** Beginning with SQLite version 3.7.3, the soft heap limit is enforced +** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] +** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], +** the soft heap limit is enforced on every memory allocation. Without +** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced +** when memory is allocated by the page cache. Testing suggests that because +** the page cache is the predominate memory user in SQLite, most +** applications will achieve adequate soft heap limit enforcement without +** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** The circumstances under which SQLite will enforce the soft heap limit may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. */ -SQLITE_API void sqlite3_soft_heap_limit(int); +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + /* ** CAPI3REF: Extract Metadata About A Column Of A Table ** ** ^This routine returns metadata about a specific column of a specific @@ -4218,38 +4416,51 @@ ** it back off again. */ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Automatically Load An Extensions -** -** ^This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new [database connections]. -** -** ^(This routine stores a pointer to the extension entry point -** in an array that is obtained from [sqlite3_malloc()]. That memory -** is deallocated by [sqlite3_reset_auto_extension()].)^ -** -** ^This function registers an extension entry point that is -** automatically invoked whenever a new [database connection] -** is opened using [sqlite3_open()], [sqlite3_open16()], -** or [sqlite3_open_v2()]. -** ^Duplicate extensions are detected so calling this routine -** multiple times with the same extension is harmless. -** ^Automatic extensions apply across all threads. +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked SQLite extension +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects and integer result as if the signature of the +** entry point where as follows: +** +**
    +**    int xEntryPoint(
    +**      sqlite3 *db,
    +**      const char **pzErrMsg,
    +**      const struct sqlite3_api_routines *pThunk
    +**    );
    +** 
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()]. */ SQLITE_API int sqlite3_auto_extension(void (*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** ^(This function disables all previously registered automatic -** extensions. It undoes the effect of all prior -** [sqlite3_auto_extension()] calls.)^ -** -** ^This function disables automatic extensions in all threads. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ SQLITE_API void sqlite3_reset_auto_extension(void); /* ** The interface to the virtual-table mechanism is currently considered @@ -4316,22 +4527,25 @@ /* ** CAPI3REF: Virtual Table Indexing Information ** KEYWORDS: sqlite3_index_info ** -** The sqlite3_index_info structure and its substructures is used to +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to ** pass information into and receive the reply from the [xBestIndex] ** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its ** results into the **Outputs** fields. ** ** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -**
    column OP expr
    +**
    column OP expr
    ** ** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is -** stored in aConstraint[].op.)^ ^(The index of the column is stored in +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in ** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint ** is usable) and false if it cannot.)^ ** ** ^The optimizer automatically inverts terms of the form "expr OP column" @@ -4387,10 +4601,19 @@ char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ }; + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros defined the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ #define SQLITE_INDEX_CONSTRAINT_EQ 2 #define SQLITE_INDEX_CONSTRAINT_GT 4 #define SQLITE_INDEX_CONSTRAINT_LE 8 #define SQLITE_INDEX_CONSTRAINT_LT 16 #define SQLITE_INDEX_CONSTRAINT_GE 32 @@ -4413,11 +4636,13 @@ ** when a new virtual table is be being created or reinitialized. ** ** ^The sqlite3_create_module_v2() interface has a fifth parameter which ** is a pointer to a destructor for the pClientData. ^SQLite will ** invoke the destructor function (if it is not NULL) when SQLite -** no longer needs the pClientData pointer. ^The sqlite3_create_module() +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. */ SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -4596,10 +4821,34 @@ sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob ); +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** +** ^This function is used to move an existing blob handle so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing blob handle to a new row can be +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + /* ** CAPI3REF: Close A BLOB Handle ** ** ^Closes an open [BLOB handle]. ** @@ -4872,11 +5121,11 @@ ** output variable when querying the system for the current mutex ** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. ** ** ^The xMutexInit method defined by this structure is invoked as ** part of system initialization by the sqlite3_initialize() function. -** ^The xMutexInit routine is calle by SQLite exactly once for each +** ^The xMutexInit routine is called by SQLite exactly once for each ** effective call to [sqlite3_initialize()]. ** ** ^The xMutexEnd method defined by this structure is invoked as ** part of system shutdown by the sqlite3_shutdown() function. The ** implementation of this method is expected to release all outstanding @@ -4905,11 +5154,11 @@ ** of passing a NULL pointer instead of a valid mutex handle are undefined ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** ** The xMutexInit() method must be threadsafe. ^It must be harmless to -** invoke xMutexInit() mutiple times within the same process and without +** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** ** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] ** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory @@ -5003,19 +5252,25 @@ ** CAPI3REF: Low-Level Control Of Database Files ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated ** with a particular database identified by the second argument. ^The -** name of the database "main" for the main database or "temp" for the +** name of the database is "main" for the main database or "temp" for the ** TEMP database, or the name that appears after the AS keyword for ** databases that are added using the [ATTACH] SQL command. ** ^A NULL pointer can be used in place of "main" to refer to the ** main database file. ** ^The third and fourth parameters to this routine ** are passed directly through to the second and third parameters of ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. +** +** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER +** case is a short-circuit path which does not actually invoke the +** underlying sqlite3_io_methods.xFileControl method. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error ** code is not remembered and will not be recalled by [sqlite3_errcode()] ** or [sqlite3_errmsg()]. The underlying xFileControl method might @@ -5069,17 +5324,18 @@ #define SQLITE_TESTCTRL_ALWAYS 13 #define SQLITE_TESTCTRL_RESERVE 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_PGHDRSZ 17 -#define SQLITE_TESTCTRL_LAST 17 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 18 +#define SQLITE_TESTCTRL_LAST 18 /* ** CAPI3REF: SQLite Runtime Status ** ** ^This interface is used to retrieve runtime status information -** about the preformance of SQLite, and optionally to reset various +** about the performance of SQLite, and optionally to reset various ** highwater marks. ^The first argument is an integer code for ** the specific parameter to measure. ^(Recognized integer codes ** are of the form [SQLITE_STATUS_MEMORY_USED | SQLITE_STATUS_...].)^ ** ^The current value of the parameter is returned into *pCurrent. ** ^The highest recorded value is returned in *pHighwater. ^If the @@ -5088,11 +5344,11 @@ ** value. For those parameters ** nothing is written into *pHighwater and the resetFlag is ignored.)^ ** ^(Other parameters record only the highwater mark and not the current ** value. For these latter parameters nothing is written into *pCurrent.)^ ** -** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** ^The sqlite3_status() routine returns SQLITE_OK on success and a ** non-zero [error code] on failure. ** ** This routine is threadsafe but is not atomic. This routine can be ** called while other threads are running the same or different SQLite ** interfaces. However the values returned in *pCurrent and @@ -5127,19 +5383,22 @@ ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** +** ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations.
    )^ +** ** ^(
    SQLITE_STATUS_PAGECACHE_USED
    **
    This parameter returns the number of pages used out of the ** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
    )^ ** ** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    **
    This parameter returns the number of bytes of page cache -** allocation which could not be statisfied by the [SQLITE_CONFIG_PAGECACHE] +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] ** buffer and where forced to overflow to [sqlite3_malloc()]. The ** returned value includes allocations that overflowed because they ** where too large (they were larger than the "sz" parameter to ** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because ** no space was left in the page cache.
    )^ @@ -5158,11 +5417,11 @@ ** outstanding at time, this parameter also reports the number of threads ** using scratch memory at the same time.)^ ** ** ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    **
    This parameter returns the number of bytes of scratch memory -** allocation which could not be statisfied by the [SQLITE_CONFIG_SCRATCH] +** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] ** buffer and where forced to overflow to [sqlite3_malloc()]. The values ** returned include overflows because the requested allocation was too ** larger (that is, because the requested allocation was larger than the ** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer ** slots were available. @@ -5188,27 +5447,31 @@ #define SQLITE_STATUS_SCRATCH_OVERFLOW 4 #define SQLITE_STATUS_MALLOC_SIZE 5 #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 #define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status ** ** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros, that -** determiness the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS_LOOKASIDE_USED | SQLITE_DBSTATUS_*] macros is likely ** to grow in future releases of SQLite. ** ** ^The current value of the requested parameter is written into *pCur ** and the highest instantaneous value is written into *pHiwtr. ^If ** the resetFlg is true, then the highest instantaneous value is ** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); @@ -5227,20 +5490,37 @@ **
    ** ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    **
    This parameter returns the number of lookaside memory slots currently ** checked out.
    )^ ** -**
    SQLITE_DBSTATUS_CACHE_USED
    -**
    ^This parameter returns the approximate number of of bytes of heap -** memory used by all pager caches associated with the database connection. +** ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. -** checked out.
    )^ +** +** ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
    **
    */ #define SQLITE_DBSTATUS_LOOKASIDE_USED 0 #define SQLITE_DBSTATUS_CACHE_USED 1 -#define SQLITE_DBSTATUS_MAX 1 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_MAX 3 /* Largest defined DBSTATUS */ /* ** CAPI3REF: Prepared Statement Status ** @@ -5315,122 +5595,134 @@ ** CAPI3REF: Application Defined Page Cache. ** KEYWORDS: {page cache} ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE], ...) interface can ** register an alternative page cache implementation by passing in an -** instance of the sqlite3_pcache_methods structure.)^ The majority of the -** heap memory used by SQLite is used by the page cache to cache data read -** from, or ready to be written to, the database file. By implementing a -** custom page cache using this API, an application can control more -** precisely the amount of memory consumed by SQLite, the way in which +** instance of the sqlite3_pcache_methods structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which ** that memory is allocated and released, and the policies used to ** determine exactly which parts of a database file are cached and for ** how long. ** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** ** ^(The contents of the sqlite3_pcache_methods structure are copied to an ** internal buffer by SQLite within the call to [sqlite3_config]. Hence ** the application may discard the parameter after the call to ** [sqlite3_config()] returns.)^ ** -** ^The xInit() method is called once for each call to [sqlite3_initialize()] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods.pArg value.)^ -** ^The xInit() method can set up up global structures and/or any mutexes +** The intent of the xInit() method is to set up global data structures ** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ ** -** ^The xShutdown() method is called from within [sqlite3_shutdown()], -** if the application invokes this API. It can be used to clean up +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up ** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. ** -** ^SQLite holds a [SQLITE_MUTEX_RECURSIVE] mutex when it invokes -** the xInit method, so the xInit method need not be threadsafe. ^The +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. All other methods must be threadsafe ** in multithreaded applications. ** ** ^SQLite will never invoke xInit() more than once without an intervening ** call to xShutdown(). ** -** ^The xCreate() method is used to construct a new cache instance. SQLite -** will typically create one cache instance for each open database file, +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will not be a power of two. ^szPage ** will the page size of the database file that is to be cached plus an -** increment (here called "R") of about 100 or 200. ^SQLite will use the +** increment (here called "R") of about 100 or 200. SQLite will use the ** extra R bytes on each page to store metadata about the underlying ** database page on disk. The value of R depends ** on the SQLite version, the target platform, and how SQLite was compiled. ** ^R is constant for a particular build of SQLite. ^The second argument to ** xCreate(), bPurgeable, is true if the cache being created will ** be used to cache database pages of a file stored on disk, or -** false if it is used for an in-memory database. ^The cache implementation +** false if it is used for an in-memory database. The cache implementation ** does not have to do anything special based with the value of bPurgeable; ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. -** ^In other words, a cache created with bPurgeable set to false will +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** ** ^(The xCachesize() method may be called at any time by SQLite to set the ** suggested maximum cache-size (number of pages stored by) the cache ** instance passed as the first argument. This is the value configured using -** the SQLite "[PRAGMA cache_size]" command.)^ ^As with the bPurgeable +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable ** parameter, the implementation is not required to do anything with this ** value; it is advisory only. ** -** ^The xPagecount() method should return the number of pages currently -** stored in the cache. +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. ** -** ^The xFetch() method is used to fetch a page and return a pointer to it. -** ^A 'page', in this context, is a buffer of szPage bytes aligned at an -** 8-byte boundary. ^The page to be fetched is determined by the key. ^The -** mimimum key value is 1. After it has been retrieved using xFetch, the page +** The xFetch() method locates a page in the cache and returns a pointer to +** the page, or a NULL pointer. +** A "page", in this context, means a buffer of szPage bytes aligned at an +** 8-byte boundary. The page to be fetched is determined by the key. ^The +** mimimum key value is 1. After it has been retrieved using xFetch, the page ** is considered to be "pinned". ** -** ^If the requested page is already in the page cache, then the page cache +** If the requested page is already in the page cache, then the page cache ** implementation must return a pointer to the page buffer with its content -** intact. ^(If the requested page is not already in the cache, then the -** behavior of the cache implementation is determined by the value of the -** createFlag parameter passed to xFetch, according to the following table: +** intact. If the requested page is not already in the cache, then the +** behavior of the cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: ** ** **
    createFlag Behaviour when page is not already in cache **
    0 Do not allocate a new page. Return NULL. **
    1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. **
    2 Make every effort to allocate a new page. Only return ** NULL if allocating a new page is effectively impossible. -**
    )^ +** ** -** SQLite will normally invoke xFetch() with a createFlag of 0 or 1. If -** a call to xFetch() with createFlag==1 returns NULL, then SQLite will +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the to xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of -** pinned pages to disk and synching the operating system disk cache. After -** attempting to unpin pages, the xFetch() method will be invoked again with -** a createFlag of 2. +** pinned pages to disk and synching the operating system disk cache. ** ** ^xUnpin() is called by SQLite with a pointer to a currently pinned page -** as its second argument. ^(If the third parameter, discard, is non-zero, -** then the page should be evicted from the cache. In this case SQLite -** assumes that the next time the page is retrieved from the cache using -** the xFetch() method, it will be zeroed.)^ ^If the discard parameter is -** zero, then the page is considered to be unpinned. ^The cache implementation +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** ^(The cache is not required to perform any reference counting. A single +** The cache must not perform any reference counting. A single ** call to xUnpin() unpins the page regardless of the number of prior calls -** to xFetch().)^ +** to xFetch(). ** -** ^The xRekey() method is used to change the key value associated with the -** page passed as the second argument from oldKey to newKey. ^If the cache -** previously contains an entry associated with newKey, it should be +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be ** discarded. ^Any prior cache entry associated with newKey is guaranteed not ** to be pinned. ** -** ^When SQLite calls the xTruncate() method, the cache must discard all +** When SQLite calls the xTruncate() method, the cache must discard all ** existing cache entries with page numbers (keys) greater than or equal -** to the value of the iLimit parameter passed to xTruncate(). ^If any +** to the value of the iLimit parameter passed to xTruncate(). If any ** of these pages are pinned, they are implicitly unpinned, meaning that ** they can be safely discarded. ** ** ^The xDestroy() method is used to delete a cache allocated by xCreate(). ** All resources associated with the specified cache should be freed. ^After @@ -5602,11 +5894,11 @@ ** ** sqlite3_backup_remaining(), sqlite3_backup_pagecount() ** ** ^Each call to sqlite3_backup_step() sets two values inside ** the [sqlite3_backup] object: the number of pages still to be backed -** up and the total number of pages in the source databae file. +** up and the total number of pages in the source database file. ** The sqlite3_backup_remaining() and sqlite3_backup_pagecount() interfaces ** retrieve these two values, respectively. ** ** ^The values returned by these functions are only updated by ** sqlite3_backup_step(). ^If the source database is modified during a backup @@ -5698,11 +5990,11 @@ ** ^(There may be at most one unlock-notify callback registered by a ** blocked connection. If sqlite3_unlock_notify() is called when the ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is cancelled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** ** The unlock-notify callback is not reentrant. If an application invokes ** any sqlite3_xxx API functions from within an unlock-notify callback, a @@ -5780,11 +6072,11 @@ /* ** CAPI3REF: String Comparison ** ** ^The [sqlite3_strnicmp()] API allows applications and extensions to ** compare the contents of two buffers containing UTF-8 strings in a -** case-indendent fashion, using the same definition of case independence +** case-independent fashion, using the same definition of case independence ** that SQLite uses internally when comparing identifiers. */ SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); /* @@ -5904,5 +6196,61 @@ #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif #endif +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry *, int nCoord, double *aCoord, int *pRes), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + double *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ +