-Dname
Define C preprocessor macro name. This macro is usable by
"%ifdef" and
"%ifndef" lines
in the grammar file.
@@ -677,10 +680,34 @@
Then the Parse() function generated will have an 4th parameter
of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of the 4th parameter
in the most recent call to Parse().
+The %extra_context directive works the same except that it
+is passed in on the ParseAlloc() or ParseInit() routines instead of
+on Parse().
+
+
+
The %extra_context directive
+
+The %extra_context directive instructs Lemon to add a 2th parameter
+to the parameter list of the ParseAlloc() and ParseInif() functions. Lemon
+doesn't do anything itself with these extra argument, but it does
+store the value make it available to C-code action routines, destructors,
+and so forth. For example, if the grammar file contains:
+
+
+ %extra_context { MyStruct *pAbc }
+
+
+Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
+of type "MyStruct*" and all action routines will have access to
+a variable named "pAbc" that is the value of that 2th parameter.
+
+The %extra_argument directive works the same except that it
+is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
+
The %fallback directive
The %fallback directive specifies an alternative meaning for one
or more tokens. The alternative meaning is tried if the original token
@@ -747,10 +774,13 @@
%include {#include <unistd.h>}
This might be needed, for example, if some of the C actions in the
grammar call functions that are prototyped in unistd.h.
+
+Use the %code directive to add code to
+the end of the generated parser.
The %left directive
The %left directive is used (along with the
ADDED ext/expert/README.md
Index: ext/expert/README.md
==================================================================
--- /dev/null
+++ ext/expert/README.md
@@ -0,0 +1,83 @@
+## SQLite Expert Extension
+
+This folder contains code for a simple system to propose useful indexes
+given a database and a set of SQL queries. It works as follows:
+
+ 1. The user database schema is copied to a temporary database.
+
+ 1. All SQL queries are prepared against the temporary database.
+ Information regarding the WHERE and ORDER BY clauses, and other query
+ features that affect index selection are recorded.
+
+ 1. The information gathered in step 2 is used to create candidate
+ indexes - indexes that the planner might have made use of in the previous
+ step, had they been available.
+
+ 1. A subset of the data in the user database is used to generate statistics
+ for all existing indexes and the candidate indexes generated in step 3
+ above.
+
+ 1. The SQL queries are prepared a second time. If the planner uses any
+ of the indexes created in step 3, they are recommended to the user.
+
+# C API
+
+The SQLite expert C API is defined in sqlite3expert.h. Most uses will proceed
+as follows:
+
+ 1. An sqlite3expert object is created by calling **sqlite3\_expert\_new()**.
+ A database handle opened by the user is passed as an argument.
+
+ 1. The sqlite3expert object is configured with one or more SQL statements
+ by making one or more calls to **sqlite3\_expert\_sql()**. Each call may
+ specify a single SQL statement, or multiple statements separated by
+ semi-colons.
+
+ 1. Optionally, the **sqlite3\_expert\_config()** API may be used to
+ configure the size of the data subset used to generate index statistics.
+ Using a smaller subset of the data can speed up the analysis.
+
+ 1. **sqlite3\_expert\_analyze()** is called to run the analysis.
+
+ 1. One or more calls are made to **sqlite3\_expert\_report()** to extract
+ components of the results of the analysis.
+
+ 1. **sqlite3\_expert\_destroy()** is called to free all resources.
+
+Refer to comments in sqlite3expert.h for further details.
+
+# sqlite3_expert application
+
+The file "expert.c" contains the code for a command line application that
+uses the API described above. It can be compiled with (for example):
+
+
+ gcc -O2 sqlite3.c expert.c sqlite3expert.c -o sqlite3_expert
+
+
+Assuming the database is named "test.db", it can then be run to analyze a
+single query:
+
+
+ ./sqlite3_expert -sql <sql-query> test.db
+
+
+Or an entire text file worth of queries with:
+
+
+ ./sqlite3_expert -file <text-file> test.db
+
+
+By default, sqlite3\_expert generates index statistics using all the data in
+the user database. For a large database, this may be prohibitively time
+consuming. The "-sample" option may be used to configure sqlite3\_expert to
+generate statistics based on an integer percentage of the user database as
+follows:
+
+
+ # Generate statistics based on 25% of the user database rows:
+ ./sqlite3_expert -sample 25 -sql <sql-query> test.db
+
+ # Do not generate any statistics at all:
+ ./sqlite3_expert -sample 0 -sql <sql-query> test.db
+
ADDED ext/expert/expert.c
Index: ext/expert/expert.c
==================================================================
--- /dev/null
+++ ext/expert/expert.c
@@ -0,0 +1,156 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include
+#include
+#include
+#include
+#include "sqlite3expert.h"
+
+
+static void option_requires_argument(const char *zOpt){
+ fprintf(stderr, "Option requires an argument: %s\n", zOpt);
+ exit(-3);
+}
+
+static int option_integer_arg(const char *zVal){
+ return atoi(zVal);
+}
+
+static void usage(char **argv){
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Usage %s ?OPTIONS? DATABASE\n", argv[0]);
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Options are:\n");
+ fprintf(stderr, " -sql SQL (analyze SQL statements passed as argument)\n");
+ fprintf(stderr, " -file FILE (read SQL statements from file FILE)\n");
+ fprintf(stderr, " -verbose LEVEL (integer verbosity level. default 1)\n");
+ fprintf(stderr, " -sample PERCENT (percent of db to sample. default 100)\n");
+ exit(-1);
+}
+
+static int readSqlFromFile(sqlite3expert *p, const char *zFile, char **pzErr){
+ FILE *in = fopen(zFile, "rb");
+ long nIn;
+ size_t nRead;
+ char *pBuf;
+ int rc;
+ if( in==0 ){
+ *pzErr = sqlite3_mprintf("failed to open file %s\n", zFile);
+ return SQLITE_ERROR;
+ }
+ fseek(in, 0, SEEK_END);
+ nIn = ftell(in);
+ rewind(in);
+ pBuf = sqlite3_malloc64( nIn+1 );
+ nRead = fread(pBuf, nIn, 1, in);
+ fclose(in);
+ if( nRead!=1 ){
+ sqlite3_free(pBuf);
+ *pzErr = sqlite3_mprintf("failed to read file %s\n", zFile);
+ return SQLITE_ERROR;
+ }
+ pBuf[nIn] = 0;
+ rc = sqlite3_expert_sql(p, pBuf, pzErr);
+ sqlite3_free(pBuf);
+ return rc;
+}
+
+int main(int argc, char **argv){
+ const char *zDb;
+ int rc = 0;
+ char *zErr = 0;
+ int i;
+ int iVerbose = 1; /* -verbose option */
+
+ sqlite3 *db = 0;
+ sqlite3expert *p = 0;
+
+ if( argc<2 ) usage(argv);
+ zDb = argv[argc-1];
+ if( zDb[0]=='-' ) usage(argv);
+ rc = sqlite3_open(zDb, &db);
+ if( rc!=SQLITE_OK ){
+ fprintf(stderr, "Cannot open db file: %s - %s\n", zDb, sqlite3_errmsg(db));
+ exit(-2);
+ }
+
+ p = sqlite3_expert_new(db, &zErr);
+ if( p==0 ){
+ fprintf(stderr, "Cannot run analysis: %s\n", zErr);
+ rc = 1;
+ }else{
+ for(i=1; i<(argc-1); i++){
+ char *zArg = argv[i];
+ int nArg;
+ if( zArg[0]=='-' && zArg[1]=='-' && zArg[2]!=0 ) zArg++;
+ nArg = (int)strlen(zArg);
+ if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-file", nArg) ){
+ if( ++i==(argc-1) ) option_requires_argument("-file");
+ rc = readSqlFromFile(p, argv[i], &zErr);
+ }
+
+ else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sql", nArg) ){
+ if( ++i==(argc-1) ) option_requires_argument("-sql");
+ rc = sqlite3_expert_sql(p, argv[i], &zErr);
+ }
+
+ else if( nArg>=3 && 0==sqlite3_strnicmp(zArg, "-sample", nArg) ){
+ int iSample;
+ if( ++i==(argc-1) ) option_requires_argument("-sample");
+ iSample = option_integer_arg(argv[i]);
+ sqlite3_expert_config(p, EXPERT_CONFIG_SAMPLE, iSample);
+ }
+
+ else if( nArg>=2 && 0==sqlite3_strnicmp(zArg, "-verbose", nArg) ){
+ if( ++i==(argc-1) ) option_requires_argument("-verbose");
+ iVerbose = option_integer_arg(argv[i]);
+ }
+
+ else{
+ usage(argv);
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_expert_analyze(p, &zErr);
+ }
+
+ if( rc==SQLITE_OK ){
+ int nQuery = sqlite3_expert_count(p);
+ if( iVerbose>0 ){
+ const char *zCand = sqlite3_expert_report(p,0,EXPERT_REPORT_CANDIDATES);
+ fprintf(stdout, "-- Candidates -------------------------------\n");
+ fprintf(stdout, "%s\n", zCand);
+ }
+ for(i=0; i0 ){
+ fprintf(stdout, "-- Query %d ----------------------------------\n",i+1);
+ fprintf(stdout, "%s\n\n", zSql);
+ }
+ fprintf(stdout, "%s\n%s\n", zIdx, zEQP);
+ }
+ }else{
+ fprintf(stderr, "Error: %s\n", zErr ? zErr : "?");
+ }
+
+ sqlite3_expert_destroy(p);
+ sqlite3_free(zErr);
+ return rc;
+}
ADDED ext/expert/expert1.test
Index: ext/expert/expert1.test
==================================================================
--- /dev/null
+++ ext/expert/expert1.test
@@ -0,0 +1,382 @@
+# 2009 Nov 11
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# The focus of this file is testing the CLI shell tool. Specifically,
+# the ".recommend" command.
+#
+#
+
+# Test plan:
+#
+#
+if {![info exists testdir]} {
+ set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
+set testprefix expert1
+
+if {[info commands sqlite3_expert_new]==""} {
+ finish_test
+ return
+}
+
+set CLI [test_binary_name sqlite3]
+set CMD [test_binary_name sqlite3_expert]
+
+proc squish {txt} {
+ regsub -all {[[:space:]]+} $txt { }
+}
+
+proc do_setup_rec_test {tn setup sql res} {
+ reset_db
+ db eval $setup
+ uplevel [list do_rec_test $tn $sql $res]
+}
+
+foreach {tn setup} {
+ 1 {
+ if {![file executable $CMD]} { continue }
+
+ proc do_rec_test {tn sql res} {
+ set res [squish [string trim $res]]
+ set tst [subst -nocommands {
+ squish [string trim [exec $::CMD -verbose 0 -sql {$sql;} test.db]]
+ }]
+ uplevel [list do_test $tn $tst $res]
+ }
+ }
+ 2 {
+ if {[info commands sqlite3_expert_new]==""} { continue }
+
+ proc do_rec_test {tn sql res} {
+ set expert [sqlite3_expert_new db]
+ $expert sql $sql
+ $expert analyze
+
+ set result [list]
+ for {set i 0} {$i < [$expert count]} {incr i} {
+ set idx [string trim [$expert report $i indexes]]
+ if {$idx==""} {set idx "(no new indexes)"}
+ lappend result $idx
+ lappend result [string trim [$expert report $i plan]]
+ }
+
+ $expert destroy
+
+ set tst [subst -nocommands {set {} [squish [join {$result}]]}]
+ uplevel [list do_test $tn $tst [string trim [squish $res]]]
+ }
+ }
+ 3 {
+ if {![file executable $CLI]} { continue }
+
+ proc do_rec_test {tn sql res} {
+ set res [squish [string trim $res]]
+ set tst [subst -nocommands {
+ squish [string trim [exec $::CLI test.db ".expert" {$sql;}]]
+ }]
+ uplevel [list do_test $tn $tst $res]
+ }
+ }
+} {
+
+ eval $setup
+
+
+do_setup_rec_test $tn.1 { CREATE TABLE t1(a, b, c) } {
+ SELECT * FROM t1
+} {
+ (no new indexes)
+ SCAN TABLE t1
+}
+
+do_setup_rec_test $tn.2 {
+ CREATE TABLE t1(a, b, c);
+} {
+ SELECT * FROM t1 WHERE b>?;
+} {
+ CREATE INDEX t1_idx_00000062 ON t1(b);
+ SEARCH TABLE t1 USING INDEX t1_idx_00000062 (b>?)
+}
+
+do_setup_rec_test $tn.3 {
+ CREATE TABLE t1(a, b, c);
+} {
+ SELECT * FROM t1 WHERE b COLLATE nocase BETWEEN ? AND ?
+} {
+ CREATE INDEX t1_idx_3e094c27 ON t1(b COLLATE NOCASE);
+ SEARCH TABLE t1 USING INDEX t1_idx_3e094c27 (b>? AND b? AND b
+#include
+#include
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+typedef sqlite3_int64 i64;
+typedef sqlite3_uint64 u64;
+
+typedef struct IdxColumn IdxColumn;
+typedef struct IdxConstraint IdxConstraint;
+typedef struct IdxScan IdxScan;
+typedef struct IdxStatement IdxStatement;
+typedef struct IdxTable IdxTable;
+typedef struct IdxWrite IdxWrite;
+
+#define STRLEN (int)strlen
+
+/*
+** A temp table name that we assume no user database will actually use.
+** If this assumption proves incorrect triggers on the table with the
+** conflicting name will be ignored.
+*/
+#define UNIQUE_TABLE_NAME "t592690916721053953805701627921227776"
+
+/*
+** A single constraint. Equivalent to either "col = ?" or "col < ?" (or
+** any other type of single-ended range constraint on a column).
+**
+** pLink:
+** Used to temporarily link IdxConstraint objects into lists while
+** creating candidate indexes.
+*/
+struct IdxConstraint {
+ char *zColl; /* Collation sequence */
+ int bRange; /* True for range, false for eq */
+ int iCol; /* Constrained table column */
+ int bFlag; /* Used by idxFindCompatible() */
+ int bDesc; /* True if ORDER BY DESC */
+ IdxConstraint *pNext; /* Next constraint in pEq or pRange list */
+ IdxConstraint *pLink; /* See above */
+};
+
+/*
+** A single scan of a single table.
+*/
+struct IdxScan {
+ IdxTable *pTab; /* Associated table object */
+ int iDb; /* Database containing table zTable */
+ i64 covering; /* Mask of columns required for cov. index */
+ IdxConstraint *pOrder; /* ORDER BY columns */
+ IdxConstraint *pEq; /* List of == constraints */
+ IdxConstraint *pRange; /* List of < constraints */
+ IdxScan *pNextScan; /* Next IdxScan object for same analysis */
+};
+
+/*
+** Information regarding a single database table. Extracted from
+** "PRAGMA table_info" by function idxGetTableInfo().
+*/
+struct IdxColumn {
+ char *zName;
+ char *zColl;
+ int iPk;
+};
+struct IdxTable {
+ int nCol;
+ char *zName; /* Table name */
+ IdxColumn *aCol;
+ IdxTable *pNext; /* Next table in linked list of all tables */
+};
+
+/*
+** An object of the following type is created for each unique table/write-op
+** seen. The objects are stored in a singly-linked list beginning at
+** sqlite3expert.pWrite.
+*/
+struct IdxWrite {
+ IdxTable *pTab;
+ int eOp; /* SQLITE_UPDATE, DELETE or INSERT */
+ IdxWrite *pNext;
+};
+
+/*
+** Each statement being analyzed is represented by an instance of this
+** structure.
+*/
+struct IdxStatement {
+ int iId; /* Statement number */
+ char *zSql; /* SQL statement */
+ char *zIdx; /* Indexes */
+ char *zEQP; /* Plan */
+ IdxStatement *pNext;
+};
+
+
+/*
+** A hash table for storing strings. With space for a payload string
+** with each entry. Methods are:
+**
+** idxHashInit()
+** idxHashClear()
+** idxHashAdd()
+** idxHashSearch()
+*/
+#define IDX_HASH_SIZE 1023
+typedef struct IdxHashEntry IdxHashEntry;
+typedef struct IdxHash IdxHash;
+struct IdxHashEntry {
+ char *zKey; /* nul-terminated key */
+ char *zVal; /* nul-terminated value string */
+ char *zVal2; /* nul-terminated value string 2 */
+ IdxHashEntry *pHashNext; /* Next entry in same hash bucket */
+ IdxHashEntry *pNext; /* Next entry in hash */
+};
+struct IdxHash {
+ IdxHashEntry *pFirst;
+ IdxHashEntry *aHash[IDX_HASH_SIZE];
+};
+
+/*
+** sqlite3expert object.
+*/
+struct sqlite3expert {
+ int iSample; /* Percentage of tables to sample for stat1 */
+ sqlite3 *db; /* User database */
+ sqlite3 *dbm; /* In-memory db for this analysis */
+ sqlite3 *dbv; /* Vtab schema for this analysis */
+ IdxTable *pTable; /* List of all IdxTable objects */
+ IdxScan *pScan; /* List of scan objects */
+ IdxWrite *pWrite; /* List of write objects */
+ IdxStatement *pStatement; /* List of IdxStatement objects */
+ int bRun; /* True once analysis has run */
+ char **pzErrmsg;
+ int rc; /* Error code from whereinfo hook */
+ IdxHash hIdx; /* Hash containing all candidate indexes */
+ char *zCandidates; /* For EXPERT_REPORT_CANDIDATES */
+};
+
+
+/*
+** Allocate and return nByte bytes of zeroed memory using sqlite3_malloc().
+** If the allocation fails, set *pRc to SQLITE_NOMEM and return NULL.
+*/
+static void *idxMalloc(int *pRc, int nByte){
+ void *pRet;
+ assert( *pRc==SQLITE_OK );
+ assert( nByte>0 );
+ pRet = sqlite3_malloc(nByte);
+ if( pRet ){
+ memset(pRet, 0, nByte);
+ }else{
+ *pRc = SQLITE_NOMEM;
+ }
+ return pRet;
+}
+
+/*
+** Initialize an IdxHash hash table.
+*/
+static void idxHashInit(IdxHash *pHash){
+ memset(pHash, 0, sizeof(IdxHash));
+}
+
+/*
+** Reset an IdxHash hash table.
+*/
+static void idxHashClear(IdxHash *pHash){
+ int i;
+ for(i=0; iaHash[i]; pEntry; pEntry=pNext){
+ pNext = pEntry->pHashNext;
+ sqlite3_free(pEntry->zVal2);
+ sqlite3_free(pEntry);
+ }
+ }
+ memset(pHash, 0, sizeof(IdxHash));
+}
+
+/*
+** Return the index of the hash bucket that the string specified by the
+** arguments to this function belongs.
+*/
+static int idxHashString(const char *z, int n){
+ unsigned int ret = 0;
+ int i;
+ for(i=0; i=0 );
+ for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+ if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+ return 1;
+ }
+ }
+ pEntry = idxMalloc(pRc, sizeof(IdxHashEntry) + nKey+1 + nVal+1);
+ if( pEntry ){
+ pEntry->zKey = (char*)&pEntry[1];
+ memcpy(pEntry->zKey, zKey, nKey);
+ if( zVal ){
+ pEntry->zVal = &pEntry->zKey[nKey+1];
+ memcpy(pEntry->zVal, zVal, nVal);
+ }
+ pEntry->pHashNext = pHash->aHash[iHash];
+ pHash->aHash[iHash] = pEntry;
+
+ pEntry->pNext = pHash->pFirst;
+ pHash->pFirst = pEntry;
+ }
+ return 0;
+}
+
+/*
+** If zKey/nKey is present in the hash table, return a pointer to the
+** hash-entry object.
+*/
+static IdxHashEntry *idxHashFind(IdxHash *pHash, const char *zKey, int nKey){
+ int iHash;
+ IdxHashEntry *pEntry;
+ if( nKey<0 ) nKey = STRLEN(zKey);
+ iHash = idxHashString(zKey, nKey);
+ assert( iHash>=0 );
+ for(pEntry=pHash->aHash[iHash]; pEntry; pEntry=pEntry->pHashNext){
+ if( STRLEN(pEntry->zKey)==nKey && 0==memcmp(pEntry->zKey, zKey, nKey) ){
+ return pEntry;
+ }
+ }
+ return 0;
+}
+
+/*
+** If the hash table contains an entry with a key equal to the string
+** passed as the final two arguments to this function, return a pointer
+** to the payload string. Otherwise, if zKey/nKey is not present in the
+** hash table, return NULL.
+*/
+static const char *idxHashSearch(IdxHash *pHash, const char *zKey, int nKey){
+ IdxHashEntry *pEntry = idxHashFind(pHash, zKey, nKey);
+ if( pEntry ) return pEntry->zVal;
+ return 0;
+}
+
+/*
+** Allocate and return a new IdxConstraint object. Set the IdxConstraint.zColl
+** variable to point to a copy of nul-terminated string zColl.
+*/
+static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
+ IdxConstraint *pNew;
+ int nColl = STRLEN(zColl);
+
+ assert( *pRc==SQLITE_OK );
+ pNew = (IdxConstraint*)idxMalloc(pRc, sizeof(IdxConstraint) * nColl + 1);
+ if( pNew ){
+ pNew->zColl = (char*)&pNew[1];
+ memcpy(pNew->zColl, zColl, nColl+1);
+ }
+ return pNew;
+}
+
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+ sqlite3 *db, /* Database handle */
+ char **pzErrmsg /* Write error here */
+){
+ *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+/*
+** Prepare an SQL statement.
+*/
+static int idxPrepareStmt(
+ sqlite3 *db, /* Database handle to compile against */
+ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
+ char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
+ const char *zSql /* SQL statement to compile */
+){
+ int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+ if( rc!=SQLITE_OK ){
+ *ppStmt = 0;
+ idxDatabaseError(db, pzErrmsg);
+ }
+ return rc;
+}
+
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
+static int idxPrintfPrepareStmt(
+ sqlite3 *db, /* Database handle to compile against */
+ sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
+ char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
+ const char *zFmt, /* printf() format of SQL statement */
+ ... /* Trailing printf() arguments */
+){
+ va_list ap;
+ int rc;
+ char *zSql;
+ va_start(ap, zFmt);
+ zSql = sqlite3_vmprintf(zFmt, ap);
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+ sqlite3_free(zSql);
+ }
+ va_end(ap);
+ return rc;
+}
+
+
+/*************************************************************************
+** Beginning of virtual table implementation.
+*/
+typedef struct ExpertVtab ExpertVtab;
+struct ExpertVtab {
+ sqlite3_vtab base;
+ IdxTable *pTab;
+ sqlite3expert *pExpert;
+};
+
+typedef struct ExpertCsr ExpertCsr;
+struct ExpertCsr {
+ sqlite3_vtab_cursor base;
+ sqlite3_stmt *pData;
+};
+
+static char *expertDequote(const char *zIn){
+ int n = STRLEN(zIn);
+ char *zRet = sqlite3_malloc(n);
+
+ assert( zIn[0]=='\'' );
+ assert( zIn[n-1]=='\'' );
+
+ if( zRet ){
+ int iOut = 0;
+ int iIn = 0;
+ for(iIn=1; iIn<(n-1); iIn++){
+ if( zIn[iIn]=='\'' ){
+ assert( zIn[iIn+1]=='\'' );
+ iIn++;
+ }
+ zRet[iOut++] = zIn[iIn];
+ }
+ zRet[iOut] = '\0';
+ }
+
+ return zRet;
+}
+
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the r-tree virtual table.
+**
+** argv[0] -> module name
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> column names...
+*/
+static int expertConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ sqlite3expert *pExpert = (sqlite3expert*)pAux;
+ ExpertVtab *p = 0;
+ int rc;
+
+ if( argc!=4 ){
+ *pzErr = sqlite3_mprintf("internal error!");
+ rc = SQLITE_ERROR;
+ }else{
+ char *zCreateTable = expertDequote(argv[3]);
+ if( zCreateTable ){
+ rc = sqlite3_declare_vtab(db, zCreateTable);
+ if( rc==SQLITE_OK ){
+ p = idxMalloc(&rc, sizeof(ExpertVtab));
+ }
+ if( rc==SQLITE_OK ){
+ p->pExpert = pExpert;
+ p->pTab = pExpert->pTable;
+ assert( sqlite3_stricmp(p->pTab->zName, argv[2])==0 );
+ }
+ sqlite3_free(zCreateTable);
+ }else{
+ rc = SQLITE_NOMEM;
+ }
+ }
+
+ *ppVtab = (sqlite3_vtab*)p;
+ return rc;
+}
+
+static int expertDisconnect(sqlite3_vtab *pVtab){
+ ExpertVtab *p = (ExpertVtab*)pVtab;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
+ ExpertVtab *p = (ExpertVtab*)pVtab;
+ int rc = SQLITE_OK;
+ int n = 0;
+ IdxScan *pScan;
+ const int opmask =
+ SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_GT |
+ SQLITE_INDEX_CONSTRAINT_LT | SQLITE_INDEX_CONSTRAINT_GE |
+ SQLITE_INDEX_CONSTRAINT_LE;
+
+ pScan = idxMalloc(&rc, sizeof(IdxScan));
+ if( pScan ){
+ int i;
+
+ /* Link the new scan object into the list */
+ pScan->pTab = p->pTab;
+ pScan->pNextScan = p->pExpert->pScan;
+ p->pExpert->pScan = pScan;
+
+ /* Add the constraints to the IdxScan object */
+ for(i=0; inConstraint; i++){
+ struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
+ if( pCons->usable
+ && pCons->iColumn>=0
+ && p->pTab->aCol[pCons->iColumn].iPk==0
+ && (pCons->op & opmask)
+ ){
+ IdxConstraint *pNew;
+ const char *zColl = sqlite3_vtab_collation(pIdxInfo, i);
+ pNew = idxNewConstraint(&rc, zColl);
+ if( pNew ){
+ pNew->iCol = pCons->iColumn;
+ if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ pNew->pNext = pScan->pEq;
+ pScan->pEq = pNew;
+ }else{
+ pNew->bRange = 1;
+ pNew->pNext = pScan->pRange;
+ pScan->pRange = pNew;
+ }
+ }
+ n++;
+ pIdxInfo->aConstraintUsage[i].argvIndex = n;
+ }
+ }
+
+ /* Add the ORDER BY to the IdxScan object */
+ for(i=pIdxInfo->nOrderBy-1; i>=0; i--){
+ int iCol = pIdxInfo->aOrderBy[i].iColumn;
+ if( iCol>=0 ){
+ IdxConstraint *pNew = idxNewConstraint(&rc, p->pTab->aCol[iCol].zColl);
+ if( pNew ){
+ pNew->iCol = iCol;
+ pNew->bDesc = pIdxInfo->aOrderBy[i].desc;
+ pNew->pNext = pScan->pOrder;
+ pNew->pLink = pScan->pOrder;
+ pScan->pOrder = pNew;
+ n++;
+ }
+ }
+ }
+ }
+
+ pIdxInfo->estimatedCost = 1000000.0 / (n+1);
+ return rc;
+}
+
+static int expertUpdate(
+ sqlite3_vtab *pVtab,
+ int nData,
+ sqlite3_value **azData,
+ sqlite_int64 *pRowid
+){
+ (void)pVtab;
+ (void)nData;
+ (void)azData;
+ (void)pRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Virtual table module xOpen method.
+*/
+static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+ int rc = SQLITE_OK;
+ ExpertCsr *pCsr;
+ (void)pVTab;
+ pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
+ *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+ return rc;
+}
+
+/*
+** Virtual table module xClose method.
+*/
+static int expertClose(sqlite3_vtab_cursor *cur){
+ ExpertCsr *pCsr = (ExpertCsr*)cur;
+ sqlite3_finalize(pCsr->pData);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Virtual table module xEof method.
+**
+** Return non-zero if the cursor does not currently point to a valid
+** record (i.e if the scan has finished), or zero otherwise.
+*/
+static int expertEof(sqlite3_vtab_cursor *cur){
+ ExpertCsr *pCsr = (ExpertCsr*)cur;
+ return pCsr->pData==0;
+}
+
+/*
+** Virtual table module xNext method.
+*/
+static int expertNext(sqlite3_vtab_cursor *cur){
+ ExpertCsr *pCsr = (ExpertCsr*)cur;
+ int rc = SQLITE_OK;
+
+ assert( pCsr->pData );
+ rc = sqlite3_step(pCsr->pData);
+ if( rc!=SQLITE_ROW ){
+ rc = sqlite3_finalize(pCsr->pData);
+ pCsr->pData = 0;
+ }else{
+ rc = SQLITE_OK;
+ }
+
+ return rc;
+}
+
+/*
+** Virtual table module xRowid method.
+*/
+static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+ (void)cur;
+ *pRowid = 0;
+ return SQLITE_OK;
+}
+
+/*
+** Virtual table module xColumn method.
+*/
+static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
+ ExpertCsr *pCsr = (ExpertCsr*)cur;
+ sqlite3_value *pVal;
+ pVal = sqlite3_column_value(pCsr->pData, i);
+ if( pVal ){
+ sqlite3_result_value(ctx, pVal);
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Virtual table module xFilter method.
+*/
+static int expertFilter(
+ sqlite3_vtab_cursor *cur,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ ExpertCsr *pCsr = (ExpertCsr*)cur;
+ ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
+ sqlite3expert *pExpert = pVtab->pExpert;
+ int rc;
+
+ (void)idxNum;
+ (void)idxStr;
+ (void)argc;
+ (void)argv;
+ rc = sqlite3_finalize(pCsr->pData);
+ pCsr->pData = 0;
+ if( rc==SQLITE_OK ){
+ rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
+ "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ rc = expertNext(cur);
+ }
+ return rc;
+}
+
+static int idxRegisterVtab(sqlite3expert *p){
+ static sqlite3_module expertModule = {
+ 2, /* iVersion */
+ expertConnect, /* xCreate - create a table */
+ expertConnect, /* xConnect - connect to an existing table */
+ expertBestIndex, /* xBestIndex - Determine search strategy */
+ expertDisconnect, /* xDisconnect - Disconnect from a table */
+ expertDisconnect, /* xDestroy - Drop a table */
+ expertOpen, /* xOpen - open a cursor */
+ expertClose, /* xClose - close a cursor */
+ expertFilter, /* xFilter - configure scan constraints */
+ expertNext, /* xNext - advance a cursor */
+ expertEof, /* xEof */
+ expertColumn, /* xColumn - read data */
+ expertRowid, /* xRowid - read data */
+ expertUpdate, /* xUpdate - write data */
+ 0, /* xBegin - begin transaction */
+ 0, /* xSync - sync transaction */
+ 0, /* xCommit - commit transaction */
+ 0, /* xRollback - rollback transaction */
+ 0, /* xFindFunction - function overloading */
+ 0, /* xRename - rename the table */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0, /* xShadowName */
+ };
+
+ return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
+}
+/*
+** End of virtual table implementation.
+*************************************************************************/
+/*
+** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
+** is called, set it to the return value of sqlite3_finalize() before
+** returning. Otherwise, discard the sqlite3_finalize() return value.
+*/
+static void idxFinalize(int *pRc, sqlite3_stmt *pStmt){
+ int rc = sqlite3_finalize(pStmt);
+ if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+/*
+** Attempt to allocate an IdxTable structure corresponding to table zTab
+** in the main database of connection db. If successful, set (*ppOut) to
+** point to the new object and return SQLITE_OK. Otherwise, return an
+** SQLite error code and set (*ppOut) to NULL. In this case *pzErrmsg may be
+** set to point to an error string.
+**
+** It is the responsibility of the caller to eventually free either the
+** IdxTable object or error message using sqlite3_free().
+*/
+static int idxGetTableInfo(
+ sqlite3 *db, /* Database connection to read details from */
+ const char *zTab, /* Table name */
+ IdxTable **ppOut, /* OUT: New object (if successful) */
+ char **pzErrmsg /* OUT: Error message (if not) */
+){
+ sqlite3_stmt *p1 = 0;
+ int nCol = 0;
+ int nTab = STRLEN(zTab);
+ int nByte = sizeof(IdxTable) + nTab + 1;
+ IdxTable *pNew = 0;
+ int rc, rc2;
+ char *pCsr = 0;
+
+ rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+ const char *zCol = (const char*)sqlite3_column_text(p1, 1);
+ nByte += 1 + STRLEN(zCol);
+ rc = sqlite3_table_column_metadata(
+ db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
+ );
+ nByte += 1 + STRLEN(zCol);
+ nCol++;
+ }
+ rc2 = sqlite3_reset(p1);
+ if( rc==SQLITE_OK ) rc = rc2;
+
+ nByte += sizeof(IdxColumn) * nCol;
+ if( rc==SQLITE_OK ){
+ pNew = idxMalloc(&rc, nByte);
+ }
+ if( rc==SQLITE_OK ){
+ pNew->aCol = (IdxColumn*)&pNew[1];
+ pNew->nCol = nCol;
+ pCsr = (char*)&pNew->aCol[nCol];
+ }
+
+ nCol = 0;
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
+ const char *zCol = (const char*)sqlite3_column_text(p1, 1);
+ int nCopy = STRLEN(zCol) + 1;
+ pNew->aCol[nCol].zName = pCsr;
+ pNew->aCol[nCol].iPk = sqlite3_column_int(p1, 5);
+ memcpy(pCsr, zCol, nCopy);
+ pCsr += nCopy;
+
+ rc = sqlite3_table_column_metadata(
+ db, "main", zTab, zCol, 0, &zCol, 0, 0, 0
+ );
+ if( rc==SQLITE_OK ){
+ nCopy = STRLEN(zCol) + 1;
+ pNew->aCol[nCol].zColl = pCsr;
+ memcpy(pCsr, zCol, nCopy);
+ pCsr += nCopy;
+ }
+
+ nCol++;
+ }
+ idxFinalize(&rc, p1);
+
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(pNew);
+ pNew = 0;
+ }else{
+ pNew->zName = pCsr;
+ memcpy(pNew->zName, zTab, nTab+1);
+ }
+
+ *ppOut = pNew;
+ return rc;
+}
+
+/*
+** This function is a no-op if *pRc is set to anything other than
+** SQLITE_OK when it is called.
+**
+** If *pRc is initially set to SQLITE_OK, then the text specified by
+** the printf() style arguments is appended to zIn and the result returned
+** in a buffer allocated by sqlite3_malloc(). sqlite3_free() is called on
+** zIn before returning.
+*/
+static char *idxAppendText(int *pRc, char *zIn, const char *zFmt, ...){
+ va_list ap;
+ char *zAppend = 0;
+ char *zRet = 0;
+ int nIn = zIn ? STRLEN(zIn) : 0;
+ int nAppend = 0;
+ va_start(ap, zFmt);
+ if( *pRc==SQLITE_OK ){
+ zAppend = sqlite3_vmprintf(zFmt, ap);
+ if( zAppend ){
+ nAppend = STRLEN(zAppend);
+ zRet = (char*)sqlite3_malloc(nIn + nAppend + 1);
+ }
+ if( zAppend && zRet ){
+ if( nIn ) memcpy(zRet, zIn, nIn);
+ memcpy(&zRet[nIn], zAppend, nAppend+1);
+ }else{
+ sqlite3_free(zRet);
+ zRet = 0;
+ *pRc = SQLITE_NOMEM;
+ }
+ sqlite3_free(zAppend);
+ sqlite3_free(zIn);
+ }
+ va_end(ap);
+ return zRet;
+}
+
+/*
+** Return true if zId must be quoted in order to use it as an SQL
+** identifier, or false otherwise.
+*/
+static int idxIdentifierRequiresQuotes(const char *zId){
+ int i;
+ for(i=0; zId[i]; i++){
+ if( !(zId[i]=='_')
+ && !(zId[i]>='0' && zId[i]<='9')
+ && !(zId[i]>='a' && zId[i]<='z')
+ && !(zId[i]>='A' && zId[i]<='Z')
+ ){
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+** This function appends an index column definition suitable for constraint
+** pCons to the string passed as zIn and returns the result.
+*/
+static char *idxAppendColDefn(
+ int *pRc, /* IN/OUT: Error code */
+ char *zIn, /* Column defn accumulated so far */
+ IdxTable *pTab, /* Table index will be created on */
+ IdxConstraint *pCons
+){
+ char *zRet = zIn;
+ IdxColumn *p = &pTab->aCol[pCons->iCol];
+ if( zRet ) zRet = idxAppendText(pRc, zRet, ", ");
+
+ if( idxIdentifierRequiresQuotes(p->zName) ){
+ zRet = idxAppendText(pRc, zRet, "%Q", p->zName);
+ }else{
+ zRet = idxAppendText(pRc, zRet, "%s", p->zName);
+ }
+
+ if( sqlite3_stricmp(p->zColl, pCons->zColl) ){
+ if( idxIdentifierRequiresQuotes(pCons->zColl) ){
+ zRet = idxAppendText(pRc, zRet, " COLLATE %Q", pCons->zColl);
+ }else{
+ zRet = idxAppendText(pRc, zRet, " COLLATE %s", pCons->zColl);
+ }
+ }
+
+ if( pCons->bDesc ){
+ zRet = idxAppendText(pRc, zRet, " DESC");
+ }
+ return zRet;
+}
+
+/*
+** Search database dbm for an index compatible with the one idxCreateFromCons()
+** would create from arguments pScan, pEq and pTail. If no error occurs and
+** such an index is found, return non-zero. Or, if no such index is found,
+** return zero.
+**
+** If an error occurs, set *pRc to an SQLite error code and return zero.
+*/
+static int idxFindCompatible(
+ int *pRc, /* OUT: Error code */
+ sqlite3* dbm, /* Database to search */
+ IdxScan *pScan, /* Scan for table to search for index on */
+ IdxConstraint *pEq, /* List of == constraints */
+ IdxConstraint *pTail /* List of range constraints */
+){
+ const char *zTbl = pScan->pTab->zName;
+ sqlite3_stmt *pIdxList = 0;
+ IdxConstraint *pIter;
+ int nEq = 0; /* Number of elements in pEq */
+ int rc;
+
+ /* Count the elements in list pEq */
+ for(pIter=pEq; pIter; pIter=pIter->pLink) nEq++;
+
+ rc = idxPrintfPrepareStmt(dbm, &pIdxList, 0, "PRAGMA index_list=%Q", zTbl);
+ while( rc==SQLITE_OK && sqlite3_step(pIdxList)==SQLITE_ROW ){
+ int bMatch = 1;
+ IdxConstraint *pT = pTail;
+ sqlite3_stmt *pInfo = 0;
+ const char *zIdx = (const char*)sqlite3_column_text(pIdxList, 1);
+
+ /* Zero the IdxConstraint.bFlag values in the pEq list */
+ for(pIter=pEq; pIter; pIter=pIter->pLink) pIter->bFlag = 0;
+
+ rc = idxPrintfPrepareStmt(dbm, &pInfo, 0, "PRAGMA index_xInfo=%Q", zIdx);
+ while( rc==SQLITE_OK && sqlite3_step(pInfo)==SQLITE_ROW ){
+ int iIdx = sqlite3_column_int(pInfo, 0);
+ int iCol = sqlite3_column_int(pInfo, 1);
+ const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+
+ if( iIdxpLink){
+ if( pIter->bFlag ) continue;
+ if( pIter->iCol!=iCol ) continue;
+ if( sqlite3_stricmp(pIter->zColl, zColl) ) continue;
+ pIter->bFlag = 1;
+ break;
+ }
+ if( pIter==0 ){
+ bMatch = 0;
+ break;
+ }
+ }else{
+ if( pT ){
+ if( pT->iCol!=iCol || sqlite3_stricmp(pT->zColl, zColl) ){
+ bMatch = 0;
+ break;
+ }
+ pT = pT->pLink;
+ }
+ }
+ }
+ idxFinalize(&rc, pInfo);
+
+ if( rc==SQLITE_OK && bMatch ){
+ sqlite3_finalize(pIdxList);
+ return 1;
+ }
+ }
+ idxFinalize(&rc, pIdxList);
+
+ *pRc = rc;
+ return 0;
+}
+
+static int idxCreateFromCons(
+ sqlite3expert *p,
+ IdxScan *pScan,
+ IdxConstraint *pEq,
+ IdxConstraint *pTail
+){
+ sqlite3 *dbm = p->dbm;
+ int rc = SQLITE_OK;
+ if( (pEq || pTail) && 0==idxFindCompatible(&rc, dbm, pScan, pEq, pTail) ){
+ IdxTable *pTab = pScan->pTab;
+ char *zCols = 0;
+ char *zIdx = 0;
+ IdxConstraint *pCons;
+ unsigned int h = 0;
+ const char *zFmt;
+
+ for(pCons=pEq; pCons; pCons=pCons->pLink){
+ zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+ }
+ for(pCons=pTail; pCons; pCons=pCons->pLink){
+ zCols = idxAppendColDefn(&rc, zCols, pTab, pCons);
+ }
+
+ if( rc==SQLITE_OK ){
+ /* Hash the list of columns to come up with a name for the index */
+ const char *zTable = pScan->pTab->zName;
+ char *zName; /* Index name */
+ int i;
+ for(i=0; zCols[i]; i++){
+ h += ((h<<3) + zCols[i]);
+ }
+ zName = sqlite3_mprintf("%s_idx_%08x", zTable, h);
+ if( zName==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ if( idxIdentifierRequiresQuotes(zTable) ){
+ zFmt = "CREATE INDEX '%q' ON %Q(%s)";
+ }else{
+ zFmt = "CREATE INDEX %s ON %s(%s)";
+ }
+ zIdx = sqlite3_mprintf(zFmt, zName, zTable, zCols);
+ if( !zIdx ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_exec(dbm, zIdx, 0, 0, p->pzErrmsg);
+ idxHashAdd(&rc, &p->hIdx, zName, zIdx);
+ }
+ sqlite3_free(zName);
+ sqlite3_free(zIdx);
+ }
+ }
+
+ sqlite3_free(zCols);
+ }
+ return rc;
+}
+
+/*
+** Return true if list pList (linked by IdxConstraint.pLink) contains
+** a constraint compatible with *p. Otherwise return false.
+*/
+static int idxFindConstraint(IdxConstraint *pList, IdxConstraint *p){
+ IdxConstraint *pCmp;
+ for(pCmp=pList; pCmp; pCmp=pCmp->pLink){
+ if( p->iCol==pCmp->iCol ) return 1;
+ }
+ return 0;
+}
+
+static int idxCreateFromWhere(
+ sqlite3expert *p,
+ IdxScan *pScan, /* Create indexes for this scan */
+ IdxConstraint *pTail /* range/ORDER BY constraints for inclusion */
+){
+ IdxConstraint *p1 = 0;
+ IdxConstraint *pCon;
+ int rc;
+
+ /* Gather up all the == constraints. */
+ for(pCon=pScan->pEq; pCon; pCon=pCon->pNext){
+ if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
+ pCon->pLink = p1;
+ p1 = pCon;
+ }
+ }
+
+ /* Create an index using the == constraints collected above. And the
+ ** range constraint/ORDER BY terms passed in by the caller, if any. */
+ rc = idxCreateFromCons(p, pScan, p1, pTail);
+
+ /* If no range/ORDER BY passed by the caller, create a version of the
+ ** index for each range constraint. */
+ if( pTail==0 ){
+ for(pCon=pScan->pRange; rc==SQLITE_OK && pCon; pCon=pCon->pNext){
+ assert( pCon->pLink==0 );
+ if( !idxFindConstraint(p1, pCon) && !idxFindConstraint(pTail, pCon) ){
+ rc = idxCreateFromCons(p, pScan, p1, pCon);
+ }
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Create candidate indexes in database [dbm] based on the data in
+** linked-list pScan.
+*/
+static int idxCreateCandidates(sqlite3expert *p){
+ int rc = SQLITE_OK;
+ IdxScan *pIter;
+
+ for(pIter=p->pScan; pIter && rc==SQLITE_OK; pIter=pIter->pNextScan){
+ rc = idxCreateFromWhere(p, pIter, 0);
+ if( rc==SQLITE_OK && pIter->pOrder ){
+ rc = idxCreateFromWhere(p, pIter, pIter->pOrder);
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Free all elements of the linked list starting at pConstraint.
+*/
+static void idxConstraintFree(IdxConstraint *pConstraint){
+ IdxConstraint *pNext;
+ IdxConstraint *p;
+
+ for(p=pConstraint; p; p=pNext){
+ pNext = p->pNext;
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Free all elements of the linked list starting from pScan up until pLast
+** (pLast is not freed).
+*/
+static void idxScanFree(IdxScan *pScan, IdxScan *pLast){
+ IdxScan *p;
+ IdxScan *pNext;
+ for(p=pScan; p!=pLast; p=pNext){
+ pNext = p->pNextScan;
+ idxConstraintFree(p->pOrder);
+ idxConstraintFree(p->pEq);
+ idxConstraintFree(p->pRange);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Free all elements of the linked list starting from pStatement up
+** until pLast (pLast is not freed).
+*/
+static void idxStatementFree(IdxStatement *pStatement, IdxStatement *pLast){
+ IdxStatement *p;
+ IdxStatement *pNext;
+ for(p=pStatement; p!=pLast; p=pNext){
+ pNext = p->pNext;
+ sqlite3_free(p->zEQP);
+ sqlite3_free(p->zIdx);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Free the linked list of IdxTable objects starting at pTab.
+*/
+static void idxTableFree(IdxTable *pTab){
+ IdxTable *pIter;
+ IdxTable *pNext;
+ for(pIter=pTab; pIter; pIter=pNext){
+ pNext = pIter->pNext;
+ sqlite3_free(pIter);
+ }
+}
+
+/*
+** Free the linked list of IdxWrite objects starting at pTab.
+*/
+static void idxWriteFree(IdxWrite *pTab){
+ IdxWrite *pIter;
+ IdxWrite *pNext;
+ for(pIter=pTab; pIter; pIter=pNext){
+ pNext = pIter->pNext;
+ sqlite3_free(pIter);
+ }
+}
+
+
+
+/*
+** This function is called after candidate indexes have been created. It
+** runs all the queries to see which indexes they prefer, and populates
+** IdxStatement.zIdx and IdxStatement.zEQP with the results.
+*/
+int idxFindIndexes(
+ sqlite3expert *p,
+ char **pzErr /* OUT: Error message (sqlite3_malloc) */
+){
+ IdxStatement *pStmt;
+ sqlite3 *dbm = p->dbm;
+ int rc = SQLITE_OK;
+
+ IdxHash hIdx;
+ idxHashInit(&hIdx);
+
+ for(pStmt=p->pStatement; rc==SQLITE_OK && pStmt; pStmt=pStmt->pNext){
+ IdxHashEntry *pEntry;
+ sqlite3_stmt *pExplain = 0;
+ idxHashClear(&hIdx);
+ rc = idxPrintfPrepareStmt(dbm, &pExplain, pzErr,
+ "EXPLAIN QUERY PLAN %s", pStmt->zSql
+ );
+ while( rc==SQLITE_OK && sqlite3_step(pExplain)==SQLITE_ROW ){
+ /* int iId = sqlite3_column_int(pExplain, 0); */
+ /* int iParent = sqlite3_column_int(pExplain, 1); */
+ /* int iNotUsed = sqlite3_column_int(pExplain, 2); */
+ const char *zDetail = (const char*)sqlite3_column_text(pExplain, 3);
+ int nDetail = STRLEN(zDetail);
+ int i;
+
+ for(i=0; ihIdx, zIdx, nIdx);
+ if( zSql ){
+ idxHashAdd(&rc, &hIdx, zSql, 0);
+ if( rc ) goto find_indexes_out;
+ }
+ break;
+ }
+ }
+
+ if( zDetail[0]!='-' ){
+ pStmt->zEQP = idxAppendText(&rc, pStmt->zEQP, "%s\n", zDetail);
+ }
+ }
+
+ for(pEntry=hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+ pStmt->zIdx = idxAppendText(&rc, pStmt->zIdx, "%s;\n", pEntry->zKey);
+ }
+
+ idxFinalize(&rc, pExplain);
+ }
+
+ find_indexes_out:
+ idxHashClear(&hIdx);
+ return rc;
+}
+
+static int idxAuthCallback(
+ void *pCtx,
+ int eOp,
+ const char *z3,
+ const char *z4,
+ const char *zDb,
+ const char *zTrigger
+){
+ int rc = SQLITE_OK;
+ (void)z4;
+ (void)zTrigger;
+ if( eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE || eOp==SQLITE_DELETE ){
+ if( sqlite3_stricmp(zDb, "main")==0 ){
+ sqlite3expert *p = (sqlite3expert*)pCtx;
+ IdxTable *pTab;
+ for(pTab=p->pTable; pTab; pTab=pTab->pNext){
+ if( 0==sqlite3_stricmp(z3, pTab->zName) ) break;
+ }
+ if( pTab ){
+ IdxWrite *pWrite;
+ for(pWrite=p->pWrite; pWrite; pWrite=pWrite->pNext){
+ if( pWrite->pTab==pTab && pWrite->eOp==eOp ) break;
+ }
+ if( pWrite==0 ){
+ pWrite = idxMalloc(&rc, sizeof(IdxWrite));
+ if( rc==SQLITE_OK ){
+ pWrite->pTab = pTab;
+ pWrite->eOp = eOp;
+ pWrite->pNext = p->pWrite;
+ p->pWrite = pWrite;
+ }
+ }
+ }
+ }
+ }
+ return rc;
+}
+
+static int idxProcessOneTrigger(
+ sqlite3expert *p,
+ IdxWrite *pWrite,
+ char **pzErr
+){
+ static const char *zInt = UNIQUE_TABLE_NAME;
+ static const char *zDrop = "DROP TABLE " UNIQUE_TABLE_NAME;
+ IdxTable *pTab = pWrite->pTab;
+ const char *zTab = pTab->zName;
+ const char *zSql =
+ "SELECT 'CREATE TEMP' || substr(sql, 7) FROM sqlite_master "
+ "WHERE tbl_name = %Q AND type IN ('table', 'trigger') "
+ "ORDER BY type;";
+ sqlite3_stmt *pSelect = 0;
+ int rc = SQLITE_OK;
+ char *zWrite = 0;
+
+ /* Create the table and its triggers in the temp schema */
+ rc = idxPrintfPrepareStmt(p->db, &pSelect, pzErr, zSql, zTab, zTab);
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSelect) ){
+ const char *zCreate = (const char*)sqlite3_column_text(pSelect, 0);
+ rc = sqlite3_exec(p->dbv, zCreate, 0, 0, pzErr);
+ }
+ idxFinalize(&rc, pSelect);
+
+ /* Rename the table in the temp schema to zInt */
+ if( rc==SQLITE_OK ){
+ char *z = sqlite3_mprintf("ALTER TABLE temp.%Q RENAME TO %Q", zTab, zInt);
+ if( z==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_exec(p->dbv, z, 0, 0, pzErr);
+ sqlite3_free(z);
+ }
+ }
+
+ switch( pWrite->eOp ){
+ case SQLITE_INSERT: {
+ int i;
+ zWrite = idxAppendText(&rc, zWrite, "INSERT INTO %Q VALUES(", zInt);
+ for(i=0; inCol; i++){
+ zWrite = idxAppendText(&rc, zWrite, "%s?", i==0 ? "" : ", ");
+ }
+ zWrite = idxAppendText(&rc, zWrite, ")");
+ break;
+ }
+ case SQLITE_UPDATE: {
+ int i;
+ zWrite = idxAppendText(&rc, zWrite, "UPDATE %Q SET ", zInt);
+ for(i=0; inCol; i++){
+ zWrite = idxAppendText(&rc, zWrite, "%s%Q=?", i==0 ? "" : ", ",
+ pTab->aCol[i].zName
+ );
+ }
+ break;
+ }
+ default: {
+ assert( pWrite->eOp==SQLITE_DELETE );
+ if( rc==SQLITE_OK ){
+ zWrite = sqlite3_mprintf("DELETE FROM %Q", zInt);
+ if( zWrite==0 ) rc = SQLITE_NOMEM;
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ sqlite3_stmt *pX = 0;
+ rc = sqlite3_prepare_v2(p->dbv, zWrite, -1, &pX, 0);
+ idxFinalize(&rc, pX);
+ if( rc!=SQLITE_OK ){
+ idxDatabaseError(p->dbv, pzErr);
+ }
+ }
+ sqlite3_free(zWrite);
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_exec(p->dbv, zDrop, 0, 0, pzErr);
+ }
+
+ return rc;
+}
+
+static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
+ int rc = SQLITE_OK;
+ IdxWrite *pEnd = 0;
+ IdxWrite *pFirst = p->pWrite;
+
+ while( rc==SQLITE_OK && pFirst!=pEnd ){
+ IdxWrite *pIter;
+ for(pIter=pFirst; rc==SQLITE_OK && pIter!=pEnd; pIter=pIter->pNext){
+ rc = idxProcessOneTrigger(p, pIter, pzErr);
+ }
+ pEnd = pFirst;
+ pFirst = p->pWrite;
+ }
+
+ return rc;
+}
+
+
+static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
+ int rc = idxRegisterVtab(p);
+ sqlite3_stmt *pSchema = 0;
+
+ /* For each table in the main db schema:
+ **
+ ** 1) Add an entry to the p->pTable list, and
+ ** 2) Create the equivalent virtual table in dbv.
+ */
+ rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
+ "SELECT type, name, sql, 1 FROM sqlite_master "
+ "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
+ " UNION ALL "
+ "SELECT type, name, sql, 2 FROM sqlite_master "
+ "WHERE type = 'trigger'"
+ " AND tbl_name IN(SELECT name FROM sqlite_master WHERE type = 'view') "
+ "ORDER BY 4, 1"
+ );
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
+ const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
+ const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
+ const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
+
+ if( zType[0]=='v' || zType[1]=='r' ){
+ rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
+ }else{
+ IdxTable *pTab;
+ rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
+ if( rc==SQLITE_OK ){
+ int i;
+ char *zInner = 0;
+ char *zOuter = 0;
+ pTab->pNext = p->pTable;
+ p->pTable = pTab;
+
+ /* The statement the vtab will pass to sqlite3_declare_vtab() */
+ zInner = idxAppendText(&rc, 0, "CREATE TABLE x(");
+ for(i=0; inCol; i++){
+ zInner = idxAppendText(&rc, zInner, "%s%Q COLLATE %s",
+ (i==0 ? "" : ", "), pTab->aCol[i].zName, pTab->aCol[i].zColl
+ );
+ }
+ zInner = idxAppendText(&rc, zInner, ")");
+
+ /* The CVT statement to create the vtab */
+ zOuter = idxAppendText(&rc, 0,
+ "CREATE VIRTUAL TABLE %Q USING expert(%Q)", zName, zInner
+ );
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_exec(p->dbv, zOuter, 0, 0, pzErrmsg);
+ }
+ sqlite3_free(zInner);
+ sqlite3_free(zOuter);
+ }
+ }
+ }
+ idxFinalize(&rc, pSchema);
+ return rc;
+}
+
+struct IdxSampleCtx {
+ int iTarget;
+ double target; /* Target nRet/nRow value */
+ double nRow; /* Number of rows seen */
+ double nRet; /* Number of rows returned */
+};
+
+static void idxSampleFunc(
+ sqlite3_context *pCtx,
+ int argc,
+ sqlite3_value **argv
+){
+ struct IdxSampleCtx *p = (struct IdxSampleCtx*)sqlite3_user_data(pCtx);
+ int bRet;
+
+ (void)argv;
+ assert( argc==0 );
+ if( p->nRow==0.0 ){
+ bRet = 1;
+ }else{
+ bRet = (p->nRet / p->nRow) <= p->target;
+ if( bRet==0 ){
+ unsigned short rnd;
+ sqlite3_randomness(2, (void*)&rnd);
+ bRet = ((int)rnd % 100) <= p->iTarget;
+ }
+ }
+
+ sqlite3_result_int(pCtx, bRet);
+ p->nRow += 1.0;
+ p->nRet += (double)bRet;
+}
+
+struct IdxRemCtx {
+ int nSlot;
+ struct IdxRemSlot {
+ int eType; /* SQLITE_NULL, INTEGER, REAL, TEXT, BLOB */
+ i64 iVal; /* SQLITE_INTEGER value */
+ double rVal; /* SQLITE_FLOAT value */
+ int nByte; /* Bytes of space allocated at z */
+ int n; /* Size of buffer z */
+ char *z; /* SQLITE_TEXT/BLOB value */
+ } aSlot[1];
+};
+
+/*
+** Implementation of scalar function rem().
+*/
+static void idxRemFunc(
+ sqlite3_context *pCtx,
+ int argc,
+ sqlite3_value **argv
+){
+ struct IdxRemCtx *p = (struct IdxRemCtx*)sqlite3_user_data(pCtx);
+ struct IdxRemSlot *pSlot;
+ int iSlot;
+ assert( argc==2 );
+
+ iSlot = sqlite3_value_int(argv[0]);
+ assert( iSlot<=p->nSlot );
+ pSlot = &p->aSlot[iSlot];
+
+ switch( pSlot->eType ){
+ case SQLITE_NULL:
+ /* no-op */
+ break;
+
+ case SQLITE_INTEGER:
+ sqlite3_result_int64(pCtx, pSlot->iVal);
+ break;
+
+ case SQLITE_FLOAT:
+ sqlite3_result_double(pCtx, pSlot->rVal);
+ break;
+
+ case SQLITE_BLOB:
+ sqlite3_result_blob(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+ break;
+
+ case SQLITE_TEXT:
+ sqlite3_result_text(pCtx, pSlot->z, pSlot->n, SQLITE_TRANSIENT);
+ break;
+ }
+
+ pSlot->eType = sqlite3_value_type(argv[1]);
+ switch( pSlot->eType ){
+ case SQLITE_NULL:
+ /* no-op */
+ break;
+
+ case SQLITE_INTEGER:
+ pSlot->iVal = sqlite3_value_int64(argv[1]);
+ break;
+
+ case SQLITE_FLOAT:
+ pSlot->rVal = sqlite3_value_double(argv[1]);
+ break;
+
+ case SQLITE_BLOB:
+ case SQLITE_TEXT: {
+ int nByte = sqlite3_value_bytes(argv[1]);
+ if( nByte>pSlot->nByte ){
+ char *zNew = (char*)sqlite3_realloc(pSlot->z, nByte*2);
+ if( zNew==0 ){
+ sqlite3_result_error_nomem(pCtx);
+ return;
+ }
+ pSlot->nByte = nByte*2;
+ pSlot->z = zNew;
+ }
+ pSlot->n = nByte;
+ if( pSlot->eType==SQLITE_BLOB ){
+ memcpy(pSlot->z, sqlite3_value_blob(argv[1]), nByte);
+ }else{
+ memcpy(pSlot->z, sqlite3_value_text(argv[1]), nByte);
+ }
+ break;
+ }
+ }
+}
+
+static int idxLargestIndex(sqlite3 *db, int *pnMax, char **pzErr){
+ int rc = SQLITE_OK;
+ const char *zMax =
+ "SELECT max(i.seqno) FROM "
+ " sqlite_master AS s, "
+ " pragma_index_list(s.name) AS l, "
+ " pragma_index_info(l.name) AS i "
+ "WHERE s.type = 'table'";
+ sqlite3_stmt *pMax = 0;
+
+ *pnMax = 0;
+ rc = idxPrepareStmt(db, &pMax, pzErr, zMax);
+ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
+ *pnMax = sqlite3_column_int(pMax, 0) + 1;
+ }
+ idxFinalize(&rc, pMax);
+
+ return rc;
+}
+
+static int idxPopulateOneStat1(
+ sqlite3expert *p,
+ sqlite3_stmt *pIndexXInfo,
+ sqlite3_stmt *pWriteStat,
+ const char *zTab,
+ const char *zIdx,
+ char **pzErr
+){
+ char *zCols = 0;
+ char *zOrder = 0;
+ char *zQuery = 0;
+ int nCol = 0;
+ int i;
+ sqlite3_stmt *pQuery = 0;
+ int *aStat = 0;
+ int rc = SQLITE_OK;
+
+ assert( p->iSample>0 );
+
+ /* Formulate the query text */
+ sqlite3_bind_text(pIndexXInfo, 1, zIdx, -1, SQLITE_STATIC);
+ while( SQLITE_OK==rc && SQLITE_ROW==sqlite3_step(pIndexXInfo) ){
+ const char *zComma = zCols==0 ? "" : ", ";
+ const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
+ const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
+ zCols = idxAppendText(&rc, zCols,
+ "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
+ );
+ zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
+ }
+ sqlite3_reset(pIndexXInfo);
+ if( rc==SQLITE_OK ){
+ if( p->iSample==100 ){
+ zQuery = sqlite3_mprintf(
+ "SELECT %s FROM %Q x ORDER BY %s", zCols, zTab, zOrder
+ );
+ }else{
+ zQuery = sqlite3_mprintf(
+ "SELECT %s FROM temp."UNIQUE_TABLE_NAME" x ORDER BY %s", zCols, zOrder
+ );
+ }
+ }
+ sqlite3_free(zCols);
+ sqlite3_free(zOrder);
+
+ /* Formulate the query text */
+ if( rc==SQLITE_OK ){
+ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
+ rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
+ }
+ sqlite3_free(zQuery);
+
+ if( rc==SQLITE_OK ){
+ aStat = (int*)idxMalloc(&rc, sizeof(int)*(nCol+1));
+ }
+ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+ IdxHashEntry *pEntry;
+ char *zStat = 0;
+ for(i=0; i<=nCol; i++) aStat[i] = 1;
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pQuery) ){
+ aStat[0]++;
+ for(i=0; ihIdx, zIdx, STRLEN(zIdx));
+ if( pEntry ){
+ assert( pEntry->zVal2==0 );
+ pEntry->zVal2 = zStat;
+ }else{
+ sqlite3_free(zStat);
+ }
+ }
+ sqlite3_free(aStat);
+ idxFinalize(&rc, pQuery);
+
+ return rc;
+}
+
+static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
+ int rc;
+ char *zSql;
+
+ rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+ if( rc!=SQLITE_OK ) return rc;
+
+ zSql = sqlite3_mprintf(
+ "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
+ );
+ if( zSql==0 ) return SQLITE_NOMEM;
+ rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
+ sqlite3_free(zSql);
+
+ return rc;
+}
+
+/*
+** This function is called as part of sqlite3_expert_analyze(). Candidate
+** indexes have already been created in database sqlite3expert.dbm, this
+** function populates sqlite_stat1 table in the same database.
+**
+** The stat1 data is generated by querying the
+*/
+static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
+ int rc = SQLITE_OK;
+ int nMax =0;
+ struct IdxRemCtx *pCtx = 0;
+ struct IdxSampleCtx samplectx;
+ int i;
+ i64 iPrev = -100000;
+ sqlite3_stmt *pAllIndex = 0;
+ sqlite3_stmt *pIndexXInfo = 0;
+ sqlite3_stmt *pWrite = 0;
+
+ const char *zAllIndex =
+ "SELECT s.rowid, s.name, l.name FROM "
+ " sqlite_master AS s, "
+ " pragma_index_list(s.name) AS l "
+ "WHERE s.type = 'table'";
+ const char *zIndexXInfo =
+ "SELECT name, coll FROM pragma_index_xinfo(?) WHERE key";
+ const char *zWrite = "INSERT INTO sqlite_stat1 VALUES(?, ?, ?)";
+
+ /* If iSample==0, no sqlite_stat1 data is required. */
+ if( p->iSample==0 ) return SQLITE_OK;
+
+ rc = idxLargestIndex(p->dbm, &nMax, pzErr);
+ if( nMax<=0 || rc!=SQLITE_OK ) return rc;
+
+ rc = sqlite3_exec(p->dbm, "ANALYZE; PRAGMA writable_schema=1", 0, 0, 0);
+
+ if( rc==SQLITE_OK ){
+ int nByte = sizeof(struct IdxRemCtx) + (sizeof(struct IdxRemSlot) * nMax);
+ pCtx = (struct IdxRemCtx*)idxMalloc(&rc, nByte);
+ }
+
+ if( rc==SQLITE_OK ){
+ sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
+ rc = sqlite3_create_function(
+ dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+ );
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(
+ p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
+ );
+ }
+
+ if( rc==SQLITE_OK ){
+ pCtx->nSlot = nMax+1;
+ rc = idxPrepareStmt(p->dbm, &pAllIndex, pzErr, zAllIndex);
+ }
+ if( rc==SQLITE_OK ){
+ rc = idxPrepareStmt(p->dbm, &pIndexXInfo, pzErr, zIndexXInfo);
+ }
+ if( rc==SQLITE_OK ){
+ rc = idxPrepareStmt(p->dbm, &pWrite, pzErr, zWrite);
+ }
+
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pAllIndex) ){
+ i64 iRowid = sqlite3_column_int64(pAllIndex, 0);
+ const char *zTab = (const char*)sqlite3_column_text(pAllIndex, 1);
+ const char *zIdx = (const char*)sqlite3_column_text(pAllIndex, 2);
+ if( p->iSample<100 && iPrev!=iRowid ){
+ samplectx.target = (double)p->iSample / 100.0;
+ samplectx.iTarget = p->iSample;
+ samplectx.nRow = 0.0;
+ samplectx.nRet = 0.0;
+ rc = idxBuildSampleTable(p, zTab);
+ if( rc!=SQLITE_OK ) break;
+ }
+ rc = idxPopulateOneStat1(p, pIndexXInfo, pWrite, zTab, zIdx, pzErr);
+ iPrev = iRowid;
+ }
+ if( rc==SQLITE_OK && p->iSample<100 ){
+ rc = sqlite3_exec(p->dbv,
+ "DROP TABLE IF EXISTS temp." UNIQUE_TABLE_NAME, 0,0,0
+ );
+ }
+
+ idxFinalize(&rc, pAllIndex);
+ idxFinalize(&rc, pIndexXInfo);
+ idxFinalize(&rc, pWrite);
+
+ for(i=0; inSlot; i++){
+ sqlite3_free(pCtx->aSlot[i].z);
+ }
+ sqlite3_free(pCtx);
+
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_master", 0, 0, 0);
+ }
+
+ sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+ return rc;
+}
+
+/*
+** Allocate a new sqlite3expert object.
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
+ int rc = SQLITE_OK;
+ sqlite3expert *pNew;
+
+ pNew = (sqlite3expert*)idxMalloc(&rc, sizeof(sqlite3expert));
+
+ /* Open two in-memory databases to work with. The "vtab database" (dbv)
+ ** will contain a virtual table corresponding to each real table in
+ ** the user database schema, and a copy of each view. It is used to
+ ** collect information regarding the WHERE, ORDER BY and other clauses
+ ** of the user's query.
+ */
+ if( rc==SQLITE_OK ){
+ pNew->db = db;
+ pNew->iSample = 100;
+ rc = sqlite3_open(":memory:", &pNew->dbv);
+ }
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_open(":memory:", &pNew->dbm);
+ if( rc==SQLITE_OK ){
+ sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
+ }
+ }
+
+
+ /* Copy the entire schema of database [db] into [dbm]. */
+ if( rc==SQLITE_OK ){
+ sqlite3_stmt *pSql;
+ rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg,
+ "SELECT sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%%'"
+ " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
+ );
+ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
+ const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
+ rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
+ }
+ idxFinalize(&rc, pSql);
+ }
+
+ /* Create the vtab schema */
+ if( rc==SQLITE_OK ){
+ rc = idxCreateVtabSchema(pNew, pzErrmsg);
+ }
+
+ /* Register the auth callback with dbv */
+ if( rc==SQLITE_OK ){
+ sqlite3_set_authorizer(pNew->dbv, idxAuthCallback, (void*)pNew);
+ }
+
+ /* If an error has occurred, free the new object and reutrn NULL. Otherwise,
+ ** return the new sqlite3expert handle. */
+ if( rc!=SQLITE_OK ){
+ sqlite3_expert_destroy(pNew);
+ pNew = 0;
+ }
+ return pNew;
+}
+
+/*
+** Configure an sqlite3expert object.
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...){
+ int rc = SQLITE_OK;
+ va_list ap;
+ va_start(ap, op);
+ switch( op ){
+ case EXPERT_CONFIG_SAMPLE: {
+ int iVal = va_arg(ap, int);
+ if( iVal<0 ) iVal = 0;
+ if( iVal>100 ) iVal = 100;
+ p->iSample = iVal;
+ break;
+ }
+ default:
+ rc = SQLITE_NOTFOUND;
+ break;
+ }
+
+ va_end(ap);
+ return rc;
+}
+
+/*
+** Add an SQL statement to the analysis.
+*/
+int sqlite3_expert_sql(
+ sqlite3expert *p, /* From sqlite3_expert_new() */
+ const char *zSql, /* SQL statement to add */
+ char **pzErr /* OUT: Error message (if any) */
+){
+ IdxScan *pScanOrig = p->pScan;
+ IdxStatement *pStmtOrig = p->pStatement;
+ int rc = SQLITE_OK;
+ const char *zStmt = zSql;
+
+ if( p->bRun ) return SQLITE_MISUSE;
+
+ while( rc==SQLITE_OK && zStmt && zStmt[0] ){
+ sqlite3_stmt *pStmt = 0;
+ rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
+ if( rc==SQLITE_OK ){
+ if( pStmt ){
+ IdxStatement *pNew;
+ const char *z = sqlite3_sql(pStmt);
+ int n = STRLEN(z);
+ pNew = (IdxStatement*)idxMalloc(&rc, sizeof(IdxStatement) + n+1);
+ if( rc==SQLITE_OK ){
+ pNew->zSql = (char*)&pNew[1];
+ memcpy(pNew->zSql, z, n+1);
+ pNew->pNext = p->pStatement;
+ if( p->pStatement ) pNew->iId = p->pStatement->iId+1;
+ p->pStatement = pNew;
+ }
+ sqlite3_finalize(pStmt);
+ }
+ }else{
+ idxDatabaseError(p->dbv, pzErr);
+ }
+ }
+
+ if( rc!=SQLITE_OK ){
+ idxScanFree(p->pScan, pScanOrig);
+ idxStatementFree(p->pStatement, pStmtOrig);
+ p->pScan = pScanOrig;
+ p->pStatement = pStmtOrig;
+ }
+
+ return rc;
+}
+
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr){
+ int rc;
+ IdxHashEntry *pEntry;
+
+ /* Do trigger processing to collect any extra IdxScan structures */
+ rc = idxProcessTriggers(p, pzErr);
+
+ /* Create candidate indexes within the in-memory database file */
+ if( rc==SQLITE_OK ){
+ rc = idxCreateCandidates(p);
+ }
+
+ /* Generate the stat1 data */
+ if( rc==SQLITE_OK ){
+ rc = idxPopulateStat1(p, pzErr);
+ }
+
+ /* Formulate the EXPERT_REPORT_CANDIDATES text */
+ for(pEntry=p->hIdx.pFirst; pEntry; pEntry=pEntry->pNext){
+ p->zCandidates = idxAppendText(&rc, p->zCandidates,
+ "%s;%s%s\n", pEntry->zVal,
+ pEntry->zVal2 ? " -- stat1: " : "", pEntry->zVal2
+ );
+ }
+
+ /* Figure out which of the candidate indexes are preferred by the query
+ ** planner and report the results to the user. */
+ if( rc==SQLITE_OK ){
+ rc = idxFindIndexes(p, pzErr);
+ }
+
+ if( rc==SQLITE_OK ){
+ p->bRun = 1;
+ }
+ return rc;
+}
+
+/*
+** Return the total number of statements that have been added to this
+** sqlite3expert using sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert *p){
+ int nRet = 0;
+ if( p->pStatement ) nRet = p->pStatement->iId+1;
+ return nRet;
+}
+
+/*
+** Return a component of the report.
+*/
+const char *sqlite3_expert_report(sqlite3expert *p, int iStmt, int eReport){
+ const char *zRet = 0;
+ IdxStatement *pStmt;
+
+ if( p->bRun==0 ) return 0;
+ for(pStmt=p->pStatement; pStmt && pStmt->iId!=iStmt; pStmt=pStmt->pNext);
+ switch( eReport ){
+ case EXPERT_REPORT_SQL:
+ if( pStmt ) zRet = pStmt->zSql;
+ break;
+ case EXPERT_REPORT_INDEXES:
+ if( pStmt ) zRet = pStmt->zIdx;
+ break;
+ case EXPERT_REPORT_PLAN:
+ if( pStmt ) zRet = pStmt->zEQP;
+ break;
+ case EXPERT_REPORT_CANDIDATES:
+ zRet = p->zCandidates;
+ break;
+ }
+ return zRet;
+}
+
+/*
+** Free an sqlite3expert object.
+*/
+void sqlite3_expert_destroy(sqlite3expert *p){
+ if( p ){
+ sqlite3_close(p->dbm);
+ sqlite3_close(p->dbv);
+ idxScanFree(p->pScan, 0);
+ idxStatementFree(p->pStatement, 0);
+ idxTableFree(p->pTable);
+ idxWriteFree(p->pWrite);
+ idxHashClear(&p->hIdx);
+ sqlite3_free(p->zCandidates);
+ sqlite3_free(p);
+ }
+}
+
+#endif /* ifndef SQLITE_OMIT_VIRTUAL_TABLE */
ADDED ext/expert/sqlite3expert.h
Index: ext/expert/sqlite3expert.h
==================================================================
--- /dev/null
+++ ext/expert/sqlite3expert.h
@@ -0,0 +1,168 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+
+#include "sqlite3.h"
+
+typedef struct sqlite3expert sqlite3expert;
+
+/*
+** Create a new sqlite3expert object.
+**
+** If successful, a pointer to the new object is returned and (*pzErr) set
+** to NULL. Or, if an error occurs, NULL is returned and (*pzErr) set to
+** an English-language error message. In this case it is the responsibility
+** of the caller to eventually free the error message buffer using
+** sqlite3_free().
+*/
+sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErr);
+
+/*
+** Configure an sqlite3expert object.
+**
+** EXPERT_CONFIG_SAMPLE:
+** By default, sqlite3_expert_analyze() generates sqlite_stat1 data for
+** each candidate index. This involves scanning and sorting the entire
+** contents of each user database table once for each candidate index
+** associated with the table. For large databases, this can be
+** prohibitively slow. This option allows the sqlite3expert object to
+** be configured so that sqlite_stat1 data is instead generated based on a
+** subset of each table, or so that no sqlite_stat1 data is used at all.
+**
+** A single integer argument is passed to this option. If the value is less
+** than or equal to zero, then no sqlite_stat1 data is generated or used by
+** the analysis - indexes are recommended based on the database schema only.
+** Or, if the value is 100 or greater, complete sqlite_stat1 data is
+** generated for each candidate index (this is the default). Finally, if the
+** value falls between 0 and 100, then it represents the percentage of user
+** table rows that should be considered when generating sqlite_stat1 data.
+**
+** Examples:
+**
+** // Do not generate any sqlite_stat1 data
+** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 0);
+**
+** // Generate sqlite_stat1 data based on 10% of the rows in each table.
+** sqlite3_expert_config(pExpert, EXPERT_CONFIG_SAMPLE, 10);
+*/
+int sqlite3_expert_config(sqlite3expert *p, int op, ...);
+
+#define EXPERT_CONFIG_SAMPLE 1 /* int */
+
+/*
+** Specify zero or more SQL statements to be included in the analysis.
+**
+** Buffer zSql must contain zero or more complete SQL statements. This
+** function parses all statements contained in the buffer and adds them
+** to the internal list of statements to analyze. If successful, SQLITE_OK
+** is returned and (*pzErr) set to NULL. Or, if an error occurs - for example
+** due to a error in the SQL - an SQLite error code is returned and (*pzErr)
+** may be set to point to an English language error message. In this case
+** the caller is responsible for eventually freeing the error message buffer
+** using sqlite3_free().
+**
+** If an error does occur while processing one of the statements in the
+** buffer passed as the second argument, none of the statements in the
+** buffer are added to the analysis.
+**
+** This function must be called before sqlite3_expert_analyze(). If a call
+** to this function is made on an sqlite3expert object that has already
+** been passed to sqlite3_expert_analyze() SQLITE_MISUSE is returned
+** immediately and no statements are added to the analysis.
+*/
+int sqlite3_expert_sql(
+ sqlite3expert *p, /* From a successful sqlite3_expert_new() */
+ const char *zSql, /* SQL statement(s) to add */
+ char **pzErr /* OUT: Error message (if any) */
+);
+
+
+/*
+** This function is called after the sqlite3expert object has been configured
+** with all SQL statements using sqlite3_expert_sql() to actually perform
+** the analysis. Once this function has been called, it is not possible to
+** add further SQL statements to the analysis.
+**
+** If successful, SQLITE_OK is returned and (*pzErr) is set to NULL. Or, if
+** an error occurs, an SQLite error code is returned and (*pzErr) set to
+** point to a buffer containing an English language error message. In this
+** case it is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** If an error does occur within this function, the sqlite3expert object
+** is no longer useful for any purpose. At that point it is no longer
+** possible to add further SQL statements to the object or to re-attempt
+** the analysis. The sqlite3expert object must still be freed using a call
+** sqlite3_expert_destroy().
+*/
+int sqlite3_expert_analyze(sqlite3expert *p, char **pzErr);
+
+/*
+** Return the total number of statements loaded using sqlite3_expert_sql().
+** The total number of SQL statements may be different from the total number
+** to calls to sqlite3_expert_sql().
+*/
+int sqlite3_expert_count(sqlite3expert*);
+
+/*
+** Return a component of the report.
+**
+** This function is called after sqlite3_expert_analyze() to extract the
+** results of the analysis. Each call to this function returns either a
+** NULL pointer or a pointer to a buffer containing a nul-terminated string.
+** The value passed as the third argument must be one of the EXPERT_REPORT_*
+** #define constants defined below.
+**
+** For some EXPERT_REPORT_* parameters, the buffer returned contains
+** information relating to a specific SQL statement. In these cases that
+** SQL statement is identified by the value passed as the second argument.
+** SQL statements are numbered from 0 in the order in which they are parsed.
+** If an out-of-range value (less than zero or equal to or greater than the
+** value returned by sqlite3_expert_count()) is passed as the second argument
+** along with such an EXPERT_REPORT_* parameter, NULL is always returned.
+**
+** EXPERT_REPORT_SQL:
+** Return the text of SQL statement iStmt.
+**
+** EXPERT_REPORT_INDEXES:
+** Return a buffer containing the CREATE INDEX statements for all recommended
+** indexes for statement iStmt. If there are no new recommeded indexes, NULL
+** is returned.
+**
+** EXPERT_REPORT_PLAN:
+** Return a buffer containing the EXPLAIN QUERY PLAN output for SQL query
+** iStmt after the proposed indexes have been added to the database schema.
+**
+** EXPERT_REPORT_CANDIDATES:
+** Return a pointer to a buffer containing the CREATE INDEX statements
+** for all indexes that were tested (for all SQL statements). The iStmt
+** parameter is ignored for EXPERT_REPORT_CANDIDATES calls.
+*/
+const char *sqlite3_expert_report(sqlite3expert*, int iStmt, int eReport);
+
+/*
+** Values for the third argument passed to sqlite3_expert_report().
+*/
+#define EXPERT_REPORT_SQL 1
+#define EXPERT_REPORT_INDEXES 2
+#define EXPERT_REPORT_PLAN 3
+#define EXPERT_REPORT_CANDIDATES 4
+
+/*
+** Free an (sqlite3expert*) handle and all associated resources. There
+** should be one call to this function for each successful call to
+** sqlite3-expert_new().
+*/
+void sqlite3_expert_destroy(sqlite3expert*);
+
+
ADDED ext/expert/test_expert.c
Index: ext/expert/test_expert.c
==================================================================
--- /dev/null
+++ ext/expert/test_expert.c
@@ -0,0 +1,220 @@
+/*
+** 2017 April 07
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#if defined(SQLITE_TEST)
+
+#include "sqlite3expert.h"
+#include
+#include
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+# define SQLITE_TCLAPI
+# endif
+#endif
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+ Tcl_CmdInfo info;
+ if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+ Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+ return TCL_ERROR;
+ }
+
+ *pDb = *(sqlite3 **)info.objClientData;
+ return TCL_OK;
+}
+
+
+/*
+** Tclcmd: $expert sql SQL
+** $expert analyze
+** $expert count
+** $expert report STMT EREPORT
+** $expert destroy
+*/
+static int SQLITE_TCLAPI testExpertCmd(
+ void *clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ sqlite3expert *pExpert = (sqlite3expert*)clientData;
+ struct Subcmd {
+ const char *zSub;
+ int nArg;
+ const char *zMsg;
+ } aSub[] = {
+ { "sql", 1, "TABLE", }, /* 0 */
+ { "analyze", 0, "", }, /* 1 */
+ { "count", 0, "", }, /* 2 */
+ { "report", 2, "STMT EREPORT", }, /* 3 */
+ { "destroy", 0, "", }, /* 4 */
+ { 0 }
+ };
+ int iSub;
+ int rc = TCL_OK;
+ char *zErr = 0;
+
+ if( objc<2 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ...");
+ return TCL_ERROR;
+ }
+ rc = Tcl_GetIndexFromObjStruct(interp,
+ objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub
+ );
+ if( rc!=TCL_OK ) return rc;
+ if( objc!=2+aSub[iSub].nArg ){
+ Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg);
+ return TCL_ERROR;
+ }
+
+ switch( iSub ){
+ case 0: { /* sql */
+ char *zArg = Tcl_GetString(objv[2]);
+ rc = sqlite3_expert_sql(pExpert, zArg, &zErr);
+ break;
+ }
+
+ case 1: { /* analyze */
+ rc = sqlite3_expert_analyze(pExpert, &zErr);
+ break;
+ }
+
+ case 2: { /* count */
+ int n = sqlite3_expert_count(pExpert);
+ Tcl_SetObjResult(interp, Tcl_NewIntObj(n));
+ break;
+ }
+
+ case 3: { /* report */
+ const char *aEnum[] = {
+ "sql", "indexes", "plan", "candidates", 0
+ };
+ int iEnum;
+ int iStmt;
+ const char *zReport;
+
+ if( Tcl_GetIntFromObj(interp, objv[2], &iStmt)
+ || Tcl_GetIndexFromObj(interp, objv[3], aEnum, "report", 0, &iEnum)
+ ){
+ return TCL_ERROR;
+ }
+
+ assert( EXPERT_REPORT_SQL==1 );
+ assert( EXPERT_REPORT_INDEXES==2 );
+ assert( EXPERT_REPORT_PLAN==3 );
+ assert( EXPERT_REPORT_CANDIDATES==4 );
+ zReport = sqlite3_expert_report(pExpert, iStmt, 1+iEnum);
+ Tcl_SetObjResult(interp, Tcl_NewStringObj(zReport, -1));
+ break;
+ }
+
+ default: /* destroy */
+ assert( iSub==4 );
+ Tcl_DeleteCommand(interp, Tcl_GetString(objv[0]));
+ break;
+ }
+
+ if( rc!=TCL_OK ){
+ if( zErr ){
+ Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1));
+ }else{
+ extern const char *sqlite3ErrName(int);
+ Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
+ }
+ }
+ sqlite3_free(zErr);
+ return rc;
+}
+
+static void SQLITE_TCLAPI testExpertDel(void *clientData){
+ sqlite3expert *pExpert = (sqlite3expert*)clientData;
+ sqlite3_expert_destroy(pExpert);
+}
+
+/*
+** sqlite3_expert_new DB
+*/
+static int SQLITE_TCLAPI test_sqlite3_expert_new(
+ void * clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ static int iCmd = 0;
+ sqlite3 *db;
+ char *zCmd = 0;
+ char *zErr = 0;
+ sqlite3expert *pExpert;
+ int rc = TCL_OK;
+
+ if( objc!=2 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "DB");
+ return TCL_ERROR;
+ }
+ if( dbHandleFromObj(interp, objv[1], &db) ){
+ return TCL_ERROR;
+ }
+
+ zCmd = sqlite3_mprintf("sqlite3expert%d", ++iCmd);
+ if( zCmd==0 ){
+ Tcl_AppendResult(interp, "out of memory", (char*)0);
+ return TCL_ERROR;
+ }
+
+ pExpert = sqlite3_expert_new(db, &zErr);
+ if( pExpert==0 ){
+ Tcl_AppendResult(interp, zErr, (char*)0);
+ rc = TCL_ERROR;
+ }else{
+ void *p = (void*)pExpert;
+ Tcl_CreateObjCommand(interp, zCmd, testExpertCmd, p, testExpertDel);
+ Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1));
+ }
+
+ sqlite3_free(zCmd);
+ sqlite3_free(zErr);
+ return rc;
+}
+
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+
+int TestExpert_Init(Tcl_Interp *interp){
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ struct Cmd {
+ const char *zCmd;
+ Tcl_ObjCmdProc *xProc;
+ } aCmd[] = {
+ { "sqlite3_expert_new", test_sqlite3_expert_new },
+ };
+ int i;
+
+ for(i=0; izCmd, p->xProc, 0, 0);
+ }
+#endif
+ return TCL_OK;
+}
+
+#endif
Index: ext/fts3/fts3.c
==================================================================
--- ext/fts3/fts3.c
+++ ext/fts3/fts3.c
@@ -1819,11 +1819,11 @@
){
int rc = SQLITE_OK; /* Return code */
const char *zCsr = zNode; /* Cursor to iterate through node */
const char *zEnd = &zCsr[nNode];/* End of interior node buffer */
char *zBuffer = 0; /* Buffer to load terms into */
- int nAlloc = 0; /* Size of allocated buffer */
+ i64 nAlloc = 0; /* Size of allocated buffer */
int isFirstTerm = 1; /* True when processing first term on page */
sqlite3_int64 iChild; /* Block id of child node to descend to */
/* Skip over the 'height' varint that occurs at the start of every
** interior node. Then load the blockid of the left-child of the b-tree
@@ -1857,18 +1857,18 @@
}
isFirstTerm = 0;
zCsr += fts3GetVarint32(zCsr, &nSuffix);
assert( nPrefix>=0 && nSuffix>=0 );
- if( &zCsr[nSuffix]>zEnd ){
+ if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr ){
rc = FTS_CORRUPT_VTAB;
goto finish_scan;
}
- if( nPrefix+nSuffix>nAlloc ){
+ if( (i64)nPrefix+nSuffix>nAlloc ){
char *zNew;
- nAlloc = (nPrefix+nSuffix) * 2;
- zNew = (char *)sqlite3_realloc(zBuffer, nAlloc);
+ nAlloc = ((i64)nPrefix+nSuffix) * 2;
+ zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc);
if( !zNew ){
rc = SQLITE_NOMEM;
goto finish_scan;
}
zBuffer = zNew;
@@ -3806,11 +3806,11 @@
*/
static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
int rc = SQLITE_OK;
UNUSED_PARAMETER(iSavepoint);
assert( ((Fts3Table *)pVtab)->inTransaction );
- assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint );
+ assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
rc = fts3SyncMethod(pVtab);
}
return rc;
@@ -3843,13 +3843,28 @@
assert( p->mxSavepoint >= iSavepoint );
TESTONLY( p->mxSavepoint = iSavepoint );
sqlite3Fts3PendingTermsClear(p);
return SQLITE_OK;
}
+
+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts3ShadowName(const char *zName){
+ static const char *azName[] = {
+ "content", "docsize", "segdir", "segments", "stat",
+ };
+ unsigned int i;
+ for(i=0; i
-/*
-** Function to query the hash-table of tokenizers (see README.tokenizers).
-*/
-static int queryTestTokenizer(
- sqlite3 *db,
- const char *zName,
- const sqlite3_tokenizer_module **pp
-){
- int rc;
- sqlite3_stmt *pStmt;
- const char zSql[] = "SELECT fts3_tokenizer(?)";
-
- *pp = 0;
- rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
- if( rc!=SQLITE_OK ){
- return rc;
- }
-
- sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
- if( SQLITE_ROW==sqlite3_step(pStmt) ){
- if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
- memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
- }
- }
-
- return sqlite3_finalize(pStmt);
-}
-
/*
** Return a pointer to a buffer containing a text representation of the
** expression passed as the first argument. The buffer is obtained from
** sqlite3_malloc(). It is the responsibility of the caller to use
** sqlite3_free() to release the memory. If an OOM condition is encountered,
@@ -1201,51 +1173,47 @@
** of a column of the fts3 table that the query expression may refer to.
** For example:
**
** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2');
*/
-static void fts3ExprTest(
+static void fts3ExprTestCommon(
+ int bRebalance,
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
- sqlite3_tokenizer_module const *pModule = 0;
sqlite3_tokenizer *pTokenizer = 0;
int rc;
char **azCol = 0;
const char *zExpr;
int nExpr;
int nCol;
int ii;
Fts3Expr *pExpr;
char *zBuf = 0;
- sqlite3 *db = sqlite3_context_db_handle(context);
+ Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context);
+ const char *zTokenizer = 0;
+ char *zErr = 0;
if( argc<3 ){
sqlite3_result_error(context,
"Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1
);
return;
}
- rc = queryTestTokenizer(db,
- (const char *)sqlite3_value_text(argv[0]), &pModule);
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }else if( !pModule ){
- sqlite3_result_error(context, "No such tokenizer module", -1);
- goto exprtest_out;
- }
-
- rc = pModule->xCreate(0, 0, &pTokenizer);
- assert( rc==SQLITE_NOMEM || rc==SQLITE_OK );
- if( rc==SQLITE_NOMEM ){
- sqlite3_result_error_nomem(context);
- goto exprtest_out;
- }
- pTokenizer->pModule = pModule;
+ zTokenizer = (const char*)sqlite3_value_text(argv[0]);
+ rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr);
+ if( rc!=SQLITE_OK ){
+ if( rc==SQLITE_NOMEM ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_error(context, zErr, -1);
+ }
+ sqlite3_free(zErr);
+ return;
+ }
zExpr = (const char *)sqlite3_value_text(argv[1]);
nExpr = sqlite3_value_bytes(argv[1]);
nCol = argc-2;
azCol = (char **)sqlite3_malloc(nCol*sizeof(char *));
@@ -1255,11 +1223,11 @@
}
for(ii=0; iixDestroy(pTokenizer);
+ if( pTokenizer ){
+ rc = pTokenizer->pModule->xDestroy(pTokenizer);
}
sqlite3_free(azCol);
}
+
+static void fts3ExprTest(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ fts3ExprTestCommon(0, context, argc, argv);
+}
+static void fts3ExprTestRebalance(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ fts3ExprTestCommon(1, context, argc, argv);
+}
/*
** Register the query expression parser test function fts3_exprtest()
** with database connection db.
*/
-int sqlite3Fts3ExprInitTestInterface(sqlite3* db){
+int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){
int rc = sqlite3_create_function(
- db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0
+ db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0
);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function(db, "fts3_exprtest_rebalance",
- -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0
+ -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0
);
}
return rc;
}
#endif
#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */
Index: ext/fts3/fts3_term.c
==================================================================
--- ext/fts3/fts3_term.c
+++ ext/fts3/fts3_term.c
@@ -359,11 +359,12 @@
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0);
return rc;
Index: ext/fts3/fts3_tokenize_vtab.c
==================================================================
--- ext/fts3/fts3_tokenize_vtab.c
+++ ext/fts3/fts3_tokenize_vtab.c
@@ -441,11 +441,12 @@
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash);
return rc;
Index: ext/fts3/fts3_write.c
==================================================================
--- ext/fts3/fts3_write.c
+++ ext/fts3/fts3_write.c
@@ -1372,19 +1372,23 @@
/* Because of the FTS3_NODE_PADDING bytes of padding, the following is
** safe (no risk of overread) even if the node data is corrupted. */
pNext += fts3GetVarint32(pNext, &nPrefix);
pNext += fts3GetVarint32(pNext, &nSuffix);
- if( nPrefix<0 || nSuffix<=0
- || &pNext[nSuffix]>&pReader->aNode[pReader->nNode]
+ if( nSuffix<=0
+ || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTermAlloc
){
return FTS_CORRUPT_VTAB;
}
- if( nPrefix+nSuffix>pReader->nTermAlloc ){
- int nNew = (nPrefix+nSuffix)*2;
- char *zNew = sqlite3_realloc(pReader->zTerm, nNew);
+ /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are
+ ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer
+ ** overflow - hence the (i64) casts. */
+ if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){
+ i64 nNew = ((i64)nPrefix+nSuffix)*2;
+ char *zNew = sqlite3_realloc64(pReader->zTerm, nNew);
if( !zNew ){
return SQLITE_NOMEM;
}
pReader->zTerm = zNew;
pReader->nTermAlloc = nNew;
@@ -1402,11 +1406,11 @@
/* Check that the doclist does not appear to extend past the end of the
** b-tree node. And that the final byte of the doclist is 0x00. If either
** of these statements is untrue, then the data structure is corrupt.
*/
- if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode]
+ if( (&pReader->aNode[pReader->nNode] - pReader->aDoclist)nDoclist
|| (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1])
){
return FTS_CORRUPT_VTAB;
}
return SQLITE_OK;
@@ -1906,10 +1910,11 @@
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pStmt, 1, iBlock);
sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 2);
}
return rc;
}
/*
@@ -1962,10 +1967,11 @@
sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free);
}
sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC);
sqlite3_step(pStmt);
rc = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 6);
}
return rc;
}
/*
@@ -3441,10 +3447,11 @@
}
sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL);
sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC);
sqlite3_step(pStmt);
*pRC = sqlite3_reset(pStmt);
+ sqlite3_bind_null(pStmt, 2);
sqlite3_free(a);
}
/*
** Merge the entire database so that there is one segment for each
@@ -3725,25 +3732,30 @@
if( bFirst==0 ){
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix);
}
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix);
+ if( nPrefix>p->iOff || nSuffix>p->nNode-p->iOff ){
+ return SQLITE_CORRUPT_VTAB;
+ }
blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc);
if( rc==SQLITE_OK ){
memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix);
p->term.n = nPrefix+nSuffix;
p->iOff += nSuffix;
if( p->iChild==0 ){
p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist);
+ if( (p->nNode-p->iOff)nDoclist ){
+ return SQLITE_CORRUPT_VTAB;
+ }
p->aDoclist = &p->aNode[p->iOff];
p->iOff += p->nDoclist;
}
}
}
assert( p->iOff<=p->nNode );
-
return rc;
}
/*
** Release all dynamic resources held by node-reader object *p.
@@ -4629,10 +4641,11 @@
sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC);
sqlite3_bind_int64(pChomp, 3, iAbsLevel);
sqlite3_bind_int(pChomp, 4, iIdx);
sqlite3_step(pChomp);
rc = sqlite3_reset(pChomp);
+ sqlite3_bind_null(pChomp, 2);
}
}
sqlite3_free(root.a);
sqlite3_free(block.a);
@@ -4708,10 +4721,11 @@
if( rc==SQLITE_OK ){
sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT);
sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
+ sqlite3_bind_null(pReplace, 2);
}
return rc;
}
@@ -5522,11 +5536,10 @@
sqlite3_value **apVal, /* Array of arguments */
sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */
){
Fts3Table *p = (Fts3Table *)pVtab;
int rc = SQLITE_OK; /* Return Code */
- int isRemove = 0; /* True for an UPDATE or DELETE */
u32 *aSzIns = 0; /* Sizes of inserted documents */
u32 *aSzDel = 0; /* Sizes of deleted documents */
int nChng = 0; /* Net change in number of documents */
int bInsertDone = 0;
@@ -5620,11 +5633,10 @@
/* If this is a DELETE or UPDATE operation, remove the old record. */
if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
- isRemove = 1;
}
/* If this is an INSERT or UPDATE operation, insert the new record. */
if( nArg>1 && rc==SQLITE_OK ){
int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
@@ -5632,11 +5644,11 @@
rc = fts3InsertData(p, apVal, pRowid);
if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
rc = FTS_CORRUPT_VTAB;
}
}
- if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
+ if( rc==SQLITE_OK ){
rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid);
}
if( rc==SQLITE_OK ){
assert( p->iPrevDocid==*pRowid );
rc = fts3InsertTerms(p, iLangid, apVal, aSzIns);
Index: ext/fts3/unicode/mkunicode.tcl
==================================================================
--- ext/fts3/unicode/mkunicode.tcl
+++ ext/fts3/unicode/mkunicode.tcl
@@ -526,10 +526,267 @@
puts ""
puts " return ret;"
puts "\}"
}
+
+proc code {txt} {
+ set txt [string trimright $txt]
+ set txt [string trimleft $txt "\n"]
+ set n [expr {[string length $txt] - [string length [string trim $txt]]}]
+ set ret ""
+ foreach L [split $txt "\n"] {
+ append ret "[string range $L $n end]\n"
+ }
+ return [uplevel "subst -nocommands {$ret}"]
+}
+
+proc intarray {lInt} {
+ set ret ""
+ set n [llength $lInt]
+ for {set i 0} {$i < $n} {incr i 10} {
+ append ret "\n "
+ foreach int [lrange $lInt $i [expr $i+9]] {
+ append ret [format "%-7s" "$int, "]
+ }
+ }
+ append ret "\n "
+ set ret
+}
+
+proc categories_switch {Cvar first lSecond} {
+ upvar $Cvar C
+ set ret ""
+ append ret "case '$first':\n"
+ append ret " switch( zCat\[1\] ){\n"
+ foreach s $lSecond {
+ append ret " case '$s': aArray\[$C($first$s)\] = 1; break;\n"
+ }
+ append ret " case '*': \n"
+ foreach s $lSecond {
+ append ret " aArray\[$C($first$s)\] = 1;\n"
+ }
+ append ret " break;\n"
+ append ret " default: return 1;"
+ append ret " }\n"
+ append ret " break;\n"
+}
+
+# Argument is a list. Each element of which is itself a list of two elements:
+#
+# * the codepoint
+# * the category
+#
+# List elements are sorted in order of codepoint.
+#
+proc print_categories {lMap} {
+ set categories {
+ Cc Cf Cn Cs
+ Ll Lm Lo Lt Lu
+ Mc Me Mn
+ Nd Nl No
+ Pc Pd Pe Pf Pi Po Ps
+ Sc Sk Sm So
+ Zl Zp Zs
+
+ LC Co
+ }
+
+ for {set i 0} {$i < [llength $categories]} {incr i} {
+ set C([lindex $categories $i]) [expr 1+$i]
+ }
+
+ set caseC [categories_switch C C {c f n s o}]
+ set caseL [categories_switch C L {l m o t u C}]
+ set caseM [categories_switch C M {c e n}]
+ set caseN [categories_switch C N {d l o}]
+ set caseP [categories_switch C P {c d e f i o s}]
+ set caseS [categories_switch C S {c k m o}]
+ set caseZ [categories_switch C Z {l p s}]
+
+ set nCat [expr [llength [array names C]] + 1]
+ puts [code {
+ int sqlite3Fts5UnicodeNCat(void) {
+ return $nCat;
+ }
+
+ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
+ aArray[0] = 1;
+ switch( zCat[0] ){
+ $caseC
+ $caseL
+ $caseM
+ $caseN
+ $caseP
+ $caseS
+ $caseZ
+ }
+ return 0;
+ }
+ }]
+
+ set nRepeat 0
+ set first [lindex $lMap 0 0]
+ set class [lindex $lMap 0 1]
+ set prev -1
+
+ set CASE(0) "Lu"
+ set CASE(1) "Ll"
+
+ foreach m $lMap {
+ foreach {codepoint cl} $m {}
+ set codepoint [expr "0x$codepoint"]
+ if {$codepoint>=(1<<20)} continue
+
+ set bNew 0
+ if {$codepoint!=($prev+1)} {
+ set bNew 1
+ } elseif {
+ $cl==$class || ($class=="LC" && $cl==$CASE([expr $nRepeat & 0x01]))
+ } {
+ incr nRepeat
+ } elseif {$class=="Lu" && $nRepeat==1 && $cl=="Ll"} {
+ set class LC
+ incr nRepeat
+ } else {
+ set bNew 1
+ }
+ if {$bNew} {
+ lappend lEntries [list $first $class $nRepeat]
+ set nRepeat 1
+ set first $codepoint
+ set class $cl
+ }
+ set prev $codepoint
+ }
+ if {$nRepeat>0} {
+ lappend lEntries [list $first $class $nRepeat]
+ }
+
+ set aBlock [list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]
+ set aMap [list]
+ foreach e $lEntries {
+ foreach {cp class nRepeat} $e {}
+ set block [expr ($cp>>16)]
+ if {$block>0 && [lindex $aBlock $block]==0} {
+ for {set i 1} {$i<=$block} {incr i} {
+ if {[lindex $aBlock $i]==0} {
+ lset aBlock $i [llength $aMap]
+ }
+ }
+ }
+ lappend aMap [expr {$cp & 0xFFFF}]
+ lappend aData [expr {($nRepeat << 5) + $C($class)}]
+ }
+ for {set i 1} {$i<[llength $aBlock]} {incr i} {
+ if {[lindex $aBlock $i]==0} {
+ lset aBlock $i [llength $aMap]
+ }
+ }
+
+ set aBlockArray [intarray $aBlock]
+ set aMapArray [intarray $aMap]
+ set aDataArray [intarray $aData]
+ puts [code {
+ static u16 aFts5UnicodeBlock[] = {$aBlockArray};
+ static u16 aFts5UnicodeMap[] = {$aMapArray};
+ static u16 aFts5UnicodeData[] = {$aDataArray};
+
+ int sqlite3Fts5UnicodeCategory(int iCode) {
+ int iRes = -1;
+ int iHi;
+ int iLo;
+ int ret;
+ u16 iKey;
+
+ if( iCode>=(1<<20) ){
+ return 0;
+ }
+ iLo = aFts5UnicodeBlock[(iCode>>16)];
+ iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+ iKey = (iCode & 0xFFFF);
+ while( iHi>iLo ){
+ int iTest = (iHi + iLo) / 2;
+ assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){
+ iRes = iTest;
+ iLo = iTest+1;
+ }else{
+ iHi = iTest;
+ }
+ }
+
+ if( iRes<0 ) return 0;
+ if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+ ret = aFts5UnicodeData[iRes] & 0x1F;
+ if( ret!=$C(LC) ) return ret;
+ return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? $C(Ll) : $C(Lu);
+ }
+
+ void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+ int i = 0;
+ int iTbl = 0;
+ while( i<128 ){
+ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+ int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+ for(; i<128 && i"
puts ""
puts "int main(int argc, char **argv)\{"
- puts " int r1, r2;"
+ puts " int r1, r2, r3;"
puts " int code;"
+ puts " r3 = 0;"
puts " r1 = isalnum_test(&code);"
puts " if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
puts " else printf(\"isalnum(): test passed\\n\");"
puts " r2 = fold_test(&code);"
puts " if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
puts " else printf(\"fold(): test passed\\n\");"
- puts " return (r1 || r2);"
+ if {$::generate_fts5_code} {
+ puts " r3 = categories_test(&code);"
+ puts " if( r3 ) printf(\"categories(): Problem with code %d\\n\",code);"
+ puts " else printf(\"categories(): test passed\\n\");"
+ }
+ puts " return (r1 || r2 || r3);"
puts "\}"
}
# Proces the command line arguments. Exit early if they are not to
# our liking.
@@ -648,15 +911,23 @@
}
}
}
print_fileheader
+
+if {$::generate_test_code} {
+ puts "typedef unsigned short int u16;"
+ puts "typedef unsigned char u8;"
+ puts "#include "
+}
# Print the isalnum() function to stdout.
#
set lRange [an_load_separator_ranges]
-print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+if {$generate_fts5_code==0} {
+ print_isalnum ${function_prefix}UnicodeIsalnum $lRange
+}
# Leave a gap between the two generated C functions.
#
puts ""
puts ""
@@ -674,21 +945,30 @@
puts ""
# Print the fold() function to stdout.
#
print_fold ${function_prefix}UnicodeFold
+
+if {$generate_fts5_code} {
+ puts ""
+ puts ""
+ print_categories [cc_load_unicodedata_text ${unicodedata.txt}]
+}
# Print the test routines and main() function to stdout, if -test
# was specified.
#
if {$::generate_test_code} {
- print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+ if {$generate_fts5_code==0} {
+ print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+ }
print_fold_test ${function_prefix}UnicodeFold $mappings
+ print_test_categories [cc_load_unicodedata_text ${unicodedata.txt}]
print_test_main
}
if {$generate_fts5_code} {
# no-op
} else {
puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
}
Index: ext/fts3/unicode/parseunicode.tcl
==================================================================
--- ext/fts3/unicode/parseunicode.tcl
+++ ext/fts3/unicode/parseunicode.tcl
@@ -140,7 +140,43 @@
set d [string trim $d]
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
}
}
+
+proc cc_load_unicodedata_text {zName} {
+ set fd [open $zName]
+ set lField {
+ code
+ character_name
+ general_category
+ canonical_combining_classes
+ bidirectional_category
+ character_decomposition_mapping
+ decimal_digit_value
+ digit_value
+ numeric_value
+ mirrored
+ unicode_1_name
+ iso10646_comment_field
+ uppercase_mapping
+ lowercase_mapping
+ titlecase_mapping
+ }
+ set lRet [list]
+
+ while { ![eof $fd] } {
+ set line [gets $fd]
+ if {$line == ""} continue
+
+ set fields [split $line ";"]
+ if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
+ foreach $lField $fields {}
+
+ lappend lRet [list $code $general_category]
+ }
+
+ close $fd
+ set lRet
+}
Index: ext/fts5/fts5.h
==================================================================
--- ext/fts5/fts5.h
+++ ext/fts5/fts5.h
@@ -442,11 +442,11 @@
** "place".
**
** This way, even if the tokenizer does not provide synonyms
** when tokenizing query text (it should not - to do would be
** inefficient), it doesn't matter if the user queries for
-** 'first + place' or '1st + place', as there are entires in the
+** 'first + place' or '1st + place', as there are entries in the
** FTS index corresponding to both forms of the first token.
**
**
** Whether it is parsing document or query text, any call to xToken that
** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit
@@ -470,11 +470,11 @@
**
** In many cases, method (1) above is the best approach. It does not add
** extra data to the FTS index or require FTS5 to query for multiple terms,
** so it is efficient in terms of disk space and query speed. However, it
** does not support prefix queries very well. If, as suggested above, the
-** token "first" is subsituted for "1st" by the tokenizer, then the query:
+** token "first" is substituted for "1st" by the tokenizer, then the query:
**
**
** ... MATCH '1s*'
**
** will not match documents that contain the token "1st" (as the tokenizer
Index: ext/fts5/fts5Int.h
==================================================================
--- ext/fts5/fts5Int.h
+++ ext/fts5/fts5Int.h
@@ -720,10 +720,12 @@
Fts5ExprPhrase *pPhrase,
Fts5Token *pToken,
int bPrefix
);
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*);
+
Fts5ExprNearset *sqlite3Fts5ParseNearset(
Fts5Parse*,
Fts5ExprNearset*,
Fts5ExprPhrase*
);
@@ -780,13 +782,16 @@
/**************************************************************************
** Interface to automatically generated code in fts5_unicode2.c.
*/
-int sqlite3Fts5UnicodeIsalnum(int c);
int sqlite3Fts5UnicodeIsdiacritic(int c);
int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic);
+
+int sqlite3Fts5UnicodeCatParse(const char*, u8*);
+int sqlite3Fts5UnicodeCategory(int iCode);
+void sqlite3Fts5UnicodeAscii(u8*, u8*);
/*
** End of interface to code in fts5_unicode2.c.
**************************************************************************/
#endif
Index: ext/fts5/fts5_aux.c
==================================================================
--- ext/fts5/fts5_aux.c
+++ ext/fts5/fts5_aux.c
@@ -355,10 +355,20 @@
*piPos = iAdj;
}
return rc;
}
+
+/*
+** Return the value in pVal interpreted as utf-8 text. Except, if pVal
+** contains a NULL value, return a pointer to a static string zero
+** bytes in length instead of a NULL pointer.
+*/
+static const char *fts5ValueToText(sqlite3_value *pVal){
+ const char *zRet = (const char*)sqlite3_value_text(pVal);
+ return zRet ? zRet : "";
+}
/*
** Implementation of snippet() function.
*/
static void fts5SnippetFunction(
@@ -391,13 +401,13 @@
}
nCol = pApi->xColumnCount(pFts);
memset(&ctx, 0, sizeof(HighlightContext));
iCol = sqlite3_value_int(apVal[0]);
- ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
- ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
- zEllips = (const char*)sqlite3_value_text(apVal[3]);
+ ctx.zOpen = fts5ValueToText(apVal[1]);
+ ctx.zClose = fts5ValueToText(apVal[2]);
+ zEllips = fts5ValueToText(apVal[3]);
nToken = sqlite3_value_int(apVal[4]);
iBestCol = (iCol>=0 ? iCol : 0);
nPhrase = pApi->xPhraseCount(pFts);
aSeen = sqlite3_malloc(nPhrase);
Index: ext/fts5/fts5_expr.c
==================================================================
--- ext/fts5/fts5_expr.c
+++ ext/fts5/fts5_expr.c
@@ -34,10 +34,11 @@
void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*);
#ifndef NDEBUG
#include
void sqlite3Fts5ParserTrace(FILE*, char*);
#endif
+int sqlite3Fts5ParserFallback(int);
struct Fts5Expr {
Fts5Index *pIndex;
Fts5Config *pConfig;
@@ -85,11 +86,12 @@
/*
** An instance of the following structure represents a single search term
** or term prefix.
*/
struct Fts5ExprTerm {
- int bPrefix; /* True for a prefix term */
+ u8 bPrefix; /* True for a prefix term */
+ u8 bFirst; /* True if token must be first in column */
char *zTerm; /* nul-terminated term */
Fts5IndexIter *pIter; /* Iterator for this term */
Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
};
@@ -166,10 +168,11 @@
case ':': tok = FTS5_COLON; break;
case ',': tok = FTS5_COMMA; break;
case '+': tok = FTS5_PLUS; break;
case '*': tok = FTS5_STAR; break;
case '-': tok = FTS5_MINUS; break;
+ case '^': tok = FTS5_CARET; break;
case '\0': tok = FTS5_EOF; break;
case '"': {
const char *z2;
tok = FTS5_STRING;
@@ -425,10 +428,11 @@
Fts5PoslistWriter writer = {0};
Fts5PoslistReader aStatic[4];
Fts5PoslistReader *aIter = aStatic;
int i;
int rc = SQLITE_OK;
+ int bFirst = pPhrase->aTerm[0].bFirst;
fts5BufferZero(&pPhrase->poslist);
/* If the aStatic[] array is not large enough, allocate a large array
** using sqlite3_malloc(). This approach could be improved upon. */
@@ -479,12 +483,14 @@
}
}
}while( bMatch==0 );
/* Append position iPos to the output */
- rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
- if( rc!=SQLITE_OK ) goto ismatch_out;
+ if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){
+ rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos);
+ if( rc!=SQLITE_OK ) goto ismatch_out;
+ }
for(i=0; inTerm; i++){
if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out;
}
}
@@ -734,11 +740,13 @@
/* Check that each phrase in the nearset matches the current row.
** Populate the pPhrase->poslist buffers at the same time. If any
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && inPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
- if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
+ if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym
+ || pNear->pColset || pPhrase->aTerm[0].bFirst
+ ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
@@ -915,10 +923,11 @@
/* Check that this node should not be FTS5_TERM */
assert( pNear->nPhrase>1
|| pNear->apPhrase[0]->nTerm>1
|| pNear->apPhrase[0]->aTerm[0].pSynonym
+ || pNear->apPhrase[0]->aTerm[0].bFirst
);
/* Initialize iLast, the "lastest" rowid any iterator points to. If the
** iterator skips through rowids in the default ascending order, this means
** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
@@ -1438,10 +1447,20 @@
}
if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
sqlite3_free(pPhrase);
}
}
+
+/*
+** Set the "bFirst" flag on the first token of the phrase passed as the
+** only argument.
+*/
+void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){
+ if( pPhrase && pPhrase->nTerm ){
+ pPhrase->aTerm[0].bFirst = 1;
+ }
+}
/*
** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated
** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is
** appended to it and the results returned.
@@ -1656,11 +1675,11 @@
if( sCtx.pPhrase==0 ){
/* This happens when parsing a token or quoted phrase that contains
** no token characters at all. (e.g ... MATCH '""'). */
sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase));
}else if( sCtx.pPhrase->nTerm ){
- sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix;
+ sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
}
pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
}
return sCtx.pPhrase;
@@ -1717,10 +1736,11 @@
0, 0);
tflags = FTS5_TOKEN_COLOCATED;
}
if( rc==SQLITE_OK ){
sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+ sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
}
}
}else{
/* This happens when parsing a token or quoted phrase that contains
** no token characters at all. (e.g ... MATCH '""'). */
@@ -1735,11 +1755,14 @@
pNew->apExprPhrase[0] = sCtx.pPhrase;
pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
pNew->pRoot->pNear->nPhrase = 1;
sCtx.pPhrase->pNode = pNew->pRoot;
- if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+ if( pOrig->nTerm==1
+ && pOrig->aTerm[0].pSynonym==0
+ && pOrig->aTerm[0].bFirst==0
+ ){
pNew->pRoot->eType = FTS5_TERM;
pNew->pRoot->xNext = fts5ExprNodeNext_TERM;
}else{
pNew->pRoot->eType = FTS5_STRING;
pNew->pRoot->xNext = fts5ExprNodeNext_STRING;
@@ -2009,10 +2032,11 @@
switch( pNode->eType ){
case FTS5_STRING: {
Fts5ExprNearset *pNear = pNode->pNear;
if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1
&& pNear->apPhrase[0]->aTerm[0].pSynonym==0
+ && pNear->apPhrase[0]->aTerm[0].bFirst==0
){
pNode->eType = FTS5_TERM;
pNode->xNext = fts5ExprNodeNext_TERM;
}else{
pNode->xNext = fts5ExprNodeNext_STRING;
@@ -2095,24 +2119,27 @@
pRet->xNext = 0;
pRet->eType = FTS5_EOF;
}
}
- if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL
- && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1)
- ){
- assert( pParse->rc==SQLITE_OK );
- pParse->rc = SQLITE_ERROR;
- assert( pParse->zErr==0 );
- pParse->zErr = sqlite3_mprintf(
- "fts5: %s queries are not supported (detail!=full)",
- pNear->nPhrase==1 ? "phrase": "NEAR"
- );
- sqlite3_free(pRet);
- pRet = 0;
- }
-
+ if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
+ if( pNear->nPhrase!=1
+ || pPhrase->nTerm>1
+ || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst)
+ ){
+ assert( pParse->rc==SQLITE_OK );
+ pParse->rc = SQLITE_ERROR;
+ assert( pParse->zErr==0 );
+ pParse->zErr = sqlite3_mprintf(
+ "fts5: %s queries are not supported (detail!=full)",
+ pNear->nPhrase==1 ? "phrase": "NEAR"
+ );
+ sqlite3_free(pRet);
+ pRet = 0;
+ }
+ }
}else{
fts5ExprAddChildren(pRet, pLeft);
fts5ExprAddChildren(pRet, pRight);
}
}
@@ -2512,18 +2539,23 @@
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
sqlite3_value **apVal /* Function arguments */
){
int iCode;
+ u8 aArr[32];
if( nArg!=1 ){
sqlite3_result_error(pCtx,
"wrong number of arguments to function fts5_isalnum", -1
);
return;
}
+ memset(aArr, 0, sizeof(aArr));
+ sqlite3Fts5UnicodeCatParse("L*", aArr);
+ sqlite3Fts5UnicodeCatParse("N*", aArr);
+ sqlite3Fts5UnicodeCatParse("Co", aArr);
iCode = sqlite3_value_int(apVal[0]);
- sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode));
+ sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]);
}
static void fts5ExprFold(
sqlite3_context *pCtx, /* Function call context */
int nArg, /* Number of args */
@@ -2563,14 +2595,16 @@
for(i=0; rc==SQLITE_OK && iz, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
}
- /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
+ /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and
+ ** sqlite3Fts5ParserFallback() are unused */
#ifndef NDEBUG
(void)sqlite3Fts5ParserTrace;
#endif
+ (void)sqlite3Fts5ParserFallback;
return rc;
}
/*
Index: ext/fts5/fts5_index.c
==================================================================
--- ext/fts5/fts5_index.c
+++ ext/fts5/fts5_index.c
@@ -756,10 +756,11 @@
sqlite3_bind_int64(p->pWriter, 1, iRowid);
sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC);
sqlite3_step(p->pWriter);
p->rc = sqlite3_reset(p->pWriter);
+ sqlite3_bind_null(p->pWriter, 2);
}
/*
** Execute the following SQL:
**
@@ -2384,10 +2385,11 @@
i64 val = sqlite3_column_int(pIdxSelect, 0);
iPg = (int)(val>>1);
bDlidx = (val & 0x0001);
}
p->rc = sqlite3_reset(pIdxSelect);
+ sqlite3_bind_null(pIdxSelect, 2);
if( iPgpgnoFirst ){
iPg = pSeg->pgnoFirst;
bDlidx = 0;
}
@@ -3596,10 +3598,11 @@
u8 aBlob[2] = {0xff, 0xff};
sqlite3_bind_int(pIdxSelect, 1, iSegid);
sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC);
assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW );
p->rc = sqlite3_reset(pIdxSelect);
+ sqlite3_bind_null(pIdxSelect, 2);
}
}
#endif
}
}
@@ -3722,10 +3725,11 @@
/* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */
sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC);
sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1));
sqlite3_step(p->pIdxWriter);
p->rc = sqlite3_reset(p->pIdxWriter);
+ sqlite3_bind_null(p->pIdxWriter, 2);
}
pWriter->iBtPage = 0;
}
/*
@@ -4907,11 +4911,17 @@
Fts5DoclistIter i1;
Fts5DoclistIter i2;
Fts5Buffer out = {0, 0, 0};
Fts5Buffer tmp = {0, 0, 0};
- if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return;
+ /* The maximum size of the output is equal to the sum of the two
+ ** input sizes + 1 varint (9 bytes). The extra varint is because if the
+ ** first rowid in one input is a large negative number, and the first in
+ ** the other a non-negative number, the delta for the non-negative
+ ** number will be larger on disk than the literal integer value
+ ** was. */
+ if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n + 9) ) return;
fts5DoclistIterInit(p1, &i1);
fts5DoclistIterInit(p2, &i2);
while( 1 ){
if( i1.iRowidn+p2->n+9) );
fts5BufferSet(&p->rc, p1, out.n, out.p);
fts5BufferFree(&tmp);
fts5BufferFree(&out);
}
@@ -5248,11 +5259,14 @@
int n = 0;
int i;
for(i=0; i=nByte ) return 0; /* Input contains fewer than nChar chars */
if( (unsigned char)p[n++]>=0xc0 ){
- while( (p[n] & 0xc0)==0x80 ) n++;
+ while( (p[n] & 0xc0)==0x80 ){
+ n++;
+ if( n>=nByte ) break;
+ }
}
}
return n;
}
@@ -5386,11 +5400,11 @@
sqlite3Fts5IterClose((Fts5IndexIter*)pRet);
pRet = 0;
fts5CloseReader(p);
}
- *ppIter = &pRet->base;
+ *ppIter = (Fts5IndexIter*)pRet;
sqlite3Fts5BufferFree(&buf);
}
return fts5IndexReturn(p);
}
Index: ext/fts5/fts5_main.c
==================================================================
--- ext/fts5/fts5_main.c
+++ ext/fts5/fts5_main.c
@@ -278,11 +278,11 @@
break;
case FTS5_SAVEPOINT:
assert( p->ts.eState==1 );
assert( iSavepoint>=0 );
- assert( iSavepoint>p->ts.iSavepoint );
+ assert( iSavepoint>=p->ts.iSavepoint );
p->ts.iSavepoint = iSavepoint;
break;
case FTS5_RELEASE:
assert( p->ts.eState==1 );
@@ -532,10 +532,16 @@
int aColMap[3];
aColMap[0] = -1;
aColMap[1] = nCol;
aColMap[2] = nCol+1;
+
+ assert( SQLITE_INDEX_CONSTRAINT_EQnConstraint; i++){
struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
int iCol = p->iColumn;
@@ -551,15 +557,15 @@
/* As there exists an unusable MATCH constraint this is an
** unusable plan. Set a prohibitively high cost. */
pInfo->estimatedCost = 1e50;
return SQLITE_OK;
}
- }else{
+ }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
int j;
for(j=1; jiCol] && p->op & pC->op && p->usable ){
+ if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
pC->iConsIndex = i;
idxFlags |= pC->fts5op;
}
}
}
@@ -1197,10 +1203,17 @@
** fts5CursorFirstSorted() above. */
assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 );
assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 );
assert( pCsr->iLastRowid==LARGEST_INT64 );
assert( pCsr->iFirstRowid==SMALLEST_INT64 );
+ if( pTab->pSortCsr->bDesc ){
+ pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid;
+ pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid;
+ }else{
+ pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
+ pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
+ }
pCsr->ePlan = FTS5_PLAN_SOURCE;
pCsr->pExpr = pTab->pSortCsr->pExpr;
rc = fts5CursorFirst(pTab, pCsr, bDesc);
}else if( pMatch ){
const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
@@ -2629,14 +2642,29 @@
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT);
}
+
+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int fts5ShadowName(const char *zName){
+ static const char *azName[] = {
+ "config", "content", "data", "docsize", "idx"
+ };
+ unsigned int i;
+ for(i=0; ip, pBuf->n, SQLITE_STATIC);
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
+ sqlite3_bind_null(pReplace, 2);
}
}
return rc;
}
@@ -1116,10 +1117,11 @@
}else{
sqlite3_bind_int(pReplace, 2, iVal);
}
sqlite3_step(pReplace);
rc = sqlite3_reset(pReplace);
+ sqlite3_bind_null(pReplace, 1);
}
if( rc==SQLITE_OK && pVal ){
int iNew = p->pConfig->iCookie + 1;
rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
if( rc==SQLITE_OK ){
Index: ext/fts5/fts5_tcl.c
==================================================================
--- ext/fts5/fts5_tcl.c
+++ ext/fts5/fts5_tcl.c
@@ -431,11 +431,11 @@
}
CASE(15, "xGetAuxdataInt") {
int iVal;
int bClear;
if( Tcl_GetBooleanFromObj(interp, objv[2], &bClear) ) return TCL_ERROR;
- iVal = ((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
+ iVal = (int)((char*)p->pApi->xGetAuxdata(p->pFts, bClear) - (char*)0);
Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
break;
}
CASE(16, "xPhraseForeach") {
@@ -480,11 +480,11 @@
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
zColvar = Tcl_GetString(objv[3]);
rc = p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
if( rc!=SQLITE_OK ){
- Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+ Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return TCL_ERROR;
}
for( ; iCol>=0; p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)){
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
rc = Tcl_EvalObjEx(interp, pScript, 0);
@@ -922,11 +922,11 @@
return TCL_ERROR;
}
rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
- return TCL_OK;
+ return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
usage:
Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
return TCL_ERROR;
}
Index: ext/fts5/fts5_test_tok.c
==================================================================
--- ext/fts5/fts5_test_tok.c
+++ ext/fts5/fts5_test_tok.c
@@ -469,11 +469,12 @@
0, /* xRollback */
0, /* xFindFunction */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
int rc; /* Return code */
rc = sqlite3_create_module(db, "fts5tokenize", &fts5tok_module, (void*)pApi);
return rc;
Index: ext/fts5/fts5_tokenize.c
==================================================================
--- ext/fts5/fts5_tokenize.c
+++ ext/fts5/fts5_tokenize.c
@@ -235,10 +235,12 @@
char *aFold; /* Buffer to fold text into */
int nFold; /* Size of aFold[] in bytes */
int bRemoveDiacritic; /* True if remove_diacritics=1 is set */
int nException;
int *aiException;
+
+ unsigned char aCategory[32]; /* True for token char categories */
};
static int fts5UnicodeAddExceptions(
Unicode61Tokenizer *p, /* Tokenizer object */
const char *z, /* Characters to treat as exceptions */
@@ -259,11 +261,11 @@
int bToken;
READ_UTF8(zCsr, zTerm, iCode);
if( iCode<128 ){
p->aTokenChar[iCode] = (unsigned char)bTokenChars;
}else{
- bToken = sqlite3Fts5UnicodeIsalnum(iCode);
+ bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)];
assert( (bToken==0 || bToken==1) );
assert( (bTokenChars==0 || bTokenChars==1) );
if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){
int i;
for(i=0; iaFold);
sqlite3_free(p);
}
return;
}
+
+static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){
+ const char *z = zCat;
+
+ while( *z ){
+ while( *z==' ' || *z=='\t' ) z++;
+ if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){
+ return SQLITE_ERROR;
+ }
+ while( *z!=' ' && *z!='\t' && *z!='\0' ) z++;
+ }
+
+ sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar);
+ return SQLITE_OK;
+}
/*
** Create a "unicode61" tokenizer.
*/
static int fts5UnicodeCreate(
@@ -338,19 +355,32 @@
if( nArg%2 ){
rc = SQLITE_ERROR;
}else{
p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer));
if( p ){
+ const char *zCat = "L* N* Co";
int i;
memset(p, 0, sizeof(Unicode61Tokenizer));
- memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
+
p->bRemoveDiacritic = 1;
p->nFold = 64;
p->aFold = sqlite3_malloc(p->nFold * sizeof(char));
if( p->aFold==0 ){
rc = SQLITE_NOMEM;
}
+
+ /* Search for a "categories" argument */
+ for(i=0; rc==SQLITE_OK && iaCategory[sqlite3Fts5UnicodeCategory(iCode)]
+ ^ fts5UnicodeIsException(p, iCode)
+ );
}
static int fts5UnicodeTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
Index: ext/fts5/fts5_unicode2.c
==================================================================
--- ext/fts5/fts5_unicode2.c
+++ ext/fts5/fts5_unicode2.c
@@ -16,139 +16,10 @@
*/
#include
-/*
-** Return true if the argument corresponds to a unicode codepoint
-** classified as either a letter or a number. Otherwise false.
-**
-** The results are undefined if the value passed to this function
-** is less than zero.
-*/
-int sqlite3Fts5UnicodeIsalnum(int c){
- /* Each unsigned integer in the following array corresponds to a contiguous
- ** range of unicode codepoints that are not either letters or numbers (i.e.
- ** codepoints for which this function should return 0).
- **
- ** The most significant 22 bits in each 32-bit value contain the first
- ** codepoint in the range. The least significant 10 bits are used to store
- ** the size of the range (always at least 1). In other words, the value
- ** ((C<<22) + N) represents a range of N codepoints starting with codepoint
- ** C. It is not possible to represent a range larger than 1023 codepoints
- ** using this format.
- */
- static const unsigned int aEntry[] = {
- 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07,
- 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01,
- 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401,
- 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01,
- 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01,
- 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802,
- 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F,
- 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401,
- 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804,
- 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403,
- 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812,
- 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001,
- 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802,
- 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805,
- 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401,
- 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03,
- 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807,
- 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001,
- 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01,
- 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804,
- 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001,
- 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802,
- 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01,
- 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06,
- 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007,
- 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006,
- 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417,
- 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14,
- 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07,
- 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01,
- 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001,
- 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802,
- 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F,
- 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002,
- 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802,
- 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006,
- 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D,
- 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802,
- 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027,
- 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403,
- 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805,
- 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04,
- 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401,
- 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005,
- 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B,
- 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A,
- 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001,
- 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59,
- 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807,
- 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01,
- 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E,
- 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100,
- 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10,
- 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402,
- 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804,
- 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012,
- 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004,
- 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002,
- 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
- 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
- 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
- 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
- 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
- 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
- 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
- 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
- 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
- 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
- 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
- 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
- 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
- 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
- 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
- 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
- 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
- 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
- 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
- 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
- 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
- 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
- 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
- 0x380400F0,
- };
- static const unsigned int aAscii[4] = {
- 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,
- };
-
- if( (unsigned int)c<128 ){
- return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 );
- }else if( (unsigned int)c<(1<<22) ){
- unsigned int key = (((unsigned int)c)<<10) | 0x000003FF;
- int iRes = 0;
- int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
- int iLo = 0;
- while( iHi>=iLo ){
- int iTest = (iHi + iLo) / 2;
- if( key >= aEntry[iTest] ){
- iRes = iTest;
- iLo = iTest+1;
- }else{
- iHi = iTest-1;
- }
- }
- assert( aEntry[0]=aEntry[iRes] );
- return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
- }
- return 1;
-}
/*
** If the argument is a codepoint corresponding to a lowercase letter
** in the ASCII range with a diacritic added, return the codepoint
@@ -356,5 +227,538 @@
ret = c + 40;
}
return ret;
}
+
+
+#if 0
+int sqlite3Fts5UnicodeNCat(void) {
+ return 32;
+}
+#endif
+
+int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
+ aArray[0] = 1;
+ switch( zCat[0] ){
+ case 'C':
+ switch( zCat[1] ){
+ case 'c': aArray[1] = 1; break;
+ case 'f': aArray[2] = 1; break;
+ case 'n': aArray[3] = 1; break;
+ case 's': aArray[4] = 1; break;
+ case 'o': aArray[31] = 1; break;
+ case '*':
+ aArray[1] = 1;
+ aArray[2] = 1;
+ aArray[3] = 1;
+ aArray[4] = 1;
+ aArray[31] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'L':
+ switch( zCat[1] ){
+ case 'l': aArray[5] = 1; break;
+ case 'm': aArray[6] = 1; break;
+ case 'o': aArray[7] = 1; break;
+ case 't': aArray[8] = 1; break;
+ case 'u': aArray[9] = 1; break;
+ case 'C': aArray[30] = 1; break;
+ case '*':
+ aArray[5] = 1;
+ aArray[6] = 1;
+ aArray[7] = 1;
+ aArray[8] = 1;
+ aArray[9] = 1;
+ aArray[30] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'M':
+ switch( zCat[1] ){
+ case 'c': aArray[10] = 1; break;
+ case 'e': aArray[11] = 1; break;
+ case 'n': aArray[12] = 1; break;
+ case '*':
+ aArray[10] = 1;
+ aArray[11] = 1;
+ aArray[12] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'N':
+ switch( zCat[1] ){
+ case 'd': aArray[13] = 1; break;
+ case 'l': aArray[14] = 1; break;
+ case 'o': aArray[15] = 1; break;
+ case '*':
+ aArray[13] = 1;
+ aArray[14] = 1;
+ aArray[15] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'P':
+ switch( zCat[1] ){
+ case 'c': aArray[16] = 1; break;
+ case 'd': aArray[17] = 1; break;
+ case 'e': aArray[18] = 1; break;
+ case 'f': aArray[19] = 1; break;
+ case 'i': aArray[20] = 1; break;
+ case 'o': aArray[21] = 1; break;
+ case 's': aArray[22] = 1; break;
+ case '*':
+ aArray[16] = 1;
+ aArray[17] = 1;
+ aArray[18] = 1;
+ aArray[19] = 1;
+ aArray[20] = 1;
+ aArray[21] = 1;
+ aArray[22] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'S':
+ switch( zCat[1] ){
+ case 'c': aArray[23] = 1; break;
+ case 'k': aArray[24] = 1; break;
+ case 'm': aArray[25] = 1; break;
+ case 'o': aArray[26] = 1; break;
+ case '*':
+ aArray[23] = 1;
+ aArray[24] = 1;
+ aArray[25] = 1;
+ aArray[26] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ case 'Z':
+ switch( zCat[1] ){
+ case 'l': aArray[27] = 1; break;
+ case 'p': aArray[28] = 1; break;
+ case 's': aArray[29] = 1; break;
+ case '*':
+ aArray[27] = 1;
+ aArray[28] = 1;
+ aArray[29] = 1;
+ break;
+ default: return 1; }
+ break;
+
+ }
+ return 0;
+}
+
+static u16 aFts5UnicodeBlock[] = {
+ 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760,
+ 1760, 1760, 1760, 1760, 1760, 1763, 1765,
+ };
+static u16 aFts5UnicodeMap[] = {
+ 0, 32, 33, 36, 37, 40, 41, 42, 43, 44,
+ 45, 46, 48, 58, 60, 63, 65, 91, 92, 93,
+ 94, 95, 96, 97, 123, 124, 125, 126, 127, 160,
+ 161, 162, 166, 167, 168, 169, 170, 171, 172, 173,
+ 174, 175, 176, 177, 178, 180, 181, 182, 184, 185,
+ 186, 187, 188, 191, 192, 215, 216, 223, 247, 248,
+ 256, 312, 313, 329, 330, 377, 383, 385, 387, 388,
+ 391, 394, 396, 398, 402, 403, 405, 406, 409, 412,
+ 414, 415, 417, 418, 423, 427, 428, 431, 434, 436,
+ 437, 440, 442, 443, 444, 446, 448, 452, 453, 454,
+ 455, 456, 457, 458, 459, 460, 461, 477, 478, 496,
+ 497, 498, 499, 500, 503, 505, 506, 564, 570, 572,
+ 573, 575, 577, 580, 583, 584, 592, 660, 661, 688,
+ 706, 710, 722, 736, 741, 748, 749, 750, 751, 768,
+ 880, 884, 885, 886, 890, 891, 894, 900, 902, 903,
+ 904, 908, 910, 912, 913, 931, 940, 975, 977, 978,
+ 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072,
+ 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369,
+ 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473,
+ 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545,
+ 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611,
+ 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758,
+ 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791,
+ 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984,
+ 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075,
+ 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210,
+ 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369,
+ 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416,
+ 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482,
+ 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519,
+ 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561,
+ 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622,
+ 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677,
+ 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749,
+ 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790,
+ 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869,
+ 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902,
+ 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947,
+ 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006,
+ 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059,
+ 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134,
+ 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199,
+ 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263,
+ 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302,
+ 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402,
+ 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458,
+ 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544,
+ 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655,
+ 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737,
+ 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773,
+ 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860,
+ 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896,
+ 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967,
+ 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046,
+ 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153,
+ 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190,
+ 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229,
+ 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295,
+ 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704,
+ 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888,
+ 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743,
+ 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906,
+ 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068,
+ 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107,
+ 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160,
+ 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435,
+ 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480,
+ 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679,
+ 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754,
+ 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824,
+ 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978,
+ 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043,
+ 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098,
+ 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168,
+ 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288,
+ 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406,
+ 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616,
+ 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976,
+ 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033,
+ 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118,
+ 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141,
+ 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184,
+ 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219,
+ 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249,
+ 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275,
+ 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317,
+ 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413,
+ 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459,
+ 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484,
+ 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500,
+ 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523,
+ 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597,
+ 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623,
+ 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972,
+ 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180,
+ 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665,
+ 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091,
+ 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101,
+ 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217,
+ 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627,
+ 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637,
+ 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647,
+ 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750,
+ 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365,
+ 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393,
+ 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520,
+ 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696,
+ 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780,
+ 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800,
+ 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812,
+ 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904,
+ 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296,
+ 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306,
+ 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317,
+ 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347,
+ 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449,
+ 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736,
+ 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938,
+ 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981,
+ 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528,
+ 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624,
+ 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800,
+ 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912,
+ 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043,
+ 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136,
+ 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264,
+ 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395,
+ 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472,
+ 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588,
+ 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643,
+ 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713,
+ 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762,
+ 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003,
+ 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203,
+ 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112,
+ 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320,
+ 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020,
+ 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075,
+ 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086,
+ 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097,
+ 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118,
+ 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279,
+ 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294,
+ 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343,
+ 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378,
+ 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490,
+ 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529,
+ 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263,
+ 311, 320, 373, 377, 394, 400, 464, 509, 640, 672,
+ 768, 800, 816, 833, 834, 842, 896, 927, 928, 968,
+ 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103,
+ 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432,
+ 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623,
+ 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912,
+ 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178,
+ 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285,
+ 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416,
+ 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760,
+ 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216,
+ 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248,
+ 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637,
+ 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298,
+ 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441,
+ 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541,
+ 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662,
+ 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922,
+ 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062,
+ 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178,
+ 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961,
+ 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003,
+ 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028,
+ 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099,
+ 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744,
+ 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368,
+ 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971,
+ 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488,
+ 1, 32, 256, 0, 65533,
+ };
+static u16 aFts5UnicodeData[] = {
+ 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53,
+ 49, 85, 333, 85, 121, 85, 841, 54, 53, 50,
+ 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61,
+ 53, 151, 58, 53, 56, 58, 39, 52, 57, 34,
+ 58, 56, 58, 57, 79, 56, 37, 85, 56, 47,
+ 39, 51, 111, 53, 745, 57, 233, 773, 57, 261,
+ 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126,
+ 126, 73, 69, 137, 37, 73, 37, 105, 101, 73,
+ 37, 73, 37, 190, 158, 37, 126, 126, 73, 37,
+ 126, 94, 37, 39, 94, 69, 135, 41, 40, 37,
+ 41, 40, 37, 41, 40, 37, 542, 37, 606, 37,
+ 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37,
+ 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582,
+ 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596,
+ 158, 38, 56, 94, 38, 101, 53, 88, 41, 53,
+ 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105,
+ 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541,
+ 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38,
+ 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76,
+ 53, 76, 53, 44, 871, 103, 85, 162, 121, 85,
+ 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684,
+ 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58,
+ 204, 70, 76, 58, 140, 71, 333, 103, 90, 39,
+ 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333,
+ 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300,
+ 38, 108, 38, 172, 501, 807, 108, 53, 39, 359,
+ 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268,
+ 138, 44, 74, 39, 236, 327, 76, 85, 333, 53,
+ 38, 199, 231, 44, 74, 263, 71, 711, 231, 39,
+ 135, 44, 39, 106, 140, 74, 74, 44, 39, 42,
+ 71, 103, 76, 333, 71, 87, 207, 58, 55, 76,
+ 42, 199, 71, 711, 231, 71, 71, 71, 44, 106,
+ 76, 76, 108, 44, 135, 39, 333, 76, 103, 44,
+ 76, 42, 295, 103, 711, 231, 71, 167, 44, 39,
+ 106, 172, 76, 42, 74, 44, 39, 71, 76, 333,
+ 53, 55, 44, 74, 263, 71, 711, 231, 71, 167,
+ 44, 39, 42, 44, 42, 140, 74, 74, 44, 44,
+ 42, 71, 103, 76, 333, 58, 39, 207, 44, 39,
+ 199, 103, 135, 71, 39, 71, 71, 103, 391, 74,
+ 44, 74, 106, 106, 44, 39, 42, 333, 111, 218,
+ 55, 58, 106, 263, 103, 743, 327, 167, 39, 108,
+ 138, 108, 140, 76, 71, 71, 76, 333, 239, 58,
+ 74, 263, 103, 743, 327, 167, 44, 39, 42, 44,
+ 170, 44, 74, 74, 76, 74, 39, 71, 76, 333,
+ 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106,
+ 44, 39, 42, 71, 76, 333, 207, 58, 199, 74,
+ 583, 775, 295, 39, 231, 44, 106, 108, 44, 266,
+ 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268,
+ 53, 333, 85, 71, 39, 71, 39, 39, 135, 231,
+ 103, 39, 39, 71, 135, 44, 71, 204, 76, 39,
+ 167, 38, 204, 333, 135, 39, 122, 501, 58, 53,
+ 122, 76, 218, 333, 335, 58, 44, 58, 44, 58,
+ 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42,
+ 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90,
+ 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76,
+ 74, 76, 39, 333, 213, 199, 74, 76, 135, 108,
+ 39, 106, 71, 234, 103, 140, 423, 44, 74, 76,
+ 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41,
+ 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319,
+ 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151,
+ 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551,
+ 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108,
+ 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76,
+ 42, 236, 266, 44, 74, 364, 117, 38, 117, 55,
+ 39, 44, 333, 335, 213, 49, 149, 108, 61, 333,
+ 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138,
+ 76, 106, 74, 44, 202, 108, 58, 85, 333, 967,
+ 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76,
+ 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44,
+ 74, 268, 202, 332, 44, 333, 333, 245, 38, 213,
+ 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44,
+ 74, 231, 333, 245, 346, 300, 314, 76, 42, 967,
+ 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415,
+ 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159,
+ 266, 268, 74, 76, 181, 333, 103, 333, 967, 198,
+ 85, 277, 108, 53, 428, 42, 236, 135, 44, 135,
+ 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260,
+ 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265,
+ 261, 265, 197, 201, 261, 41, 41, 41, 94, 229,
+ 265, 453, 261, 264, 261, 264, 261, 264, 165, 69,
+ 137, 40, 56, 37, 120, 101, 69, 137, 40, 120,
+ 133, 69, 137, 120, 261, 169, 120, 101, 69, 137,
+ 40, 88, 381, 162, 209, 85, 52, 51, 54, 84,
+ 51, 54, 52, 277, 59, 60, 162, 61, 309, 52,
+ 51, 149, 80, 117, 57, 54, 50, 373, 57, 53,
+ 48, 341, 61, 162, 194, 47, 38, 207, 121, 54,
+ 50, 38, 335, 121, 54, 50, 422, 855, 428, 139,
+ 44, 107, 396, 90, 41, 154, 41, 90, 37, 105,
+ 69, 105, 37, 58, 41, 90, 57, 169, 218, 41,
+ 58, 41, 58, 41, 58, 137, 58, 37, 137, 37,
+ 135, 37, 90, 69, 73, 185, 94, 101, 58, 57,
+ 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186,
+ 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018,
+ 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666,
+ 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217,
+ 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57,
+ 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50,
+ 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
+ 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50,
+ 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54,
+ 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
+ 50, 54, 50, 54, 50, 54, 50, 54, 50, 54,
+ 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281,
+ 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69,
+ 254, 105, 37, 94, 37, 94, 165, 70, 105, 37,
+ 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221,
+ 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231,
+ 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52,
+ 51, 117, 52, 51, 53, 52, 51, 309, 49, 85,
+ 49, 53, 52, 51, 85, 52, 51, 54, 50, 54,
+ 50, 54, 50, 54, 50, 181, 38, 341, 81, 858,
+ 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54,
+ 50, 54, 50, 54, 50, 54, 50, 54, 50, 90,
+ 54, 50, 54, 50, 54, 50, 54, 50, 49, 54,
+ 82, 58, 302, 140, 74, 49, 166, 90, 110, 38,
+ 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887,
+ 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178,
+ 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274,
+ 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38,
+ 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333,
+ 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798,
+ 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69,
+ 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382,
+ 70, 37, 231, 44, 103, 44, 135, 44, 743, 74,
+ 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74,
+ 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333,
+ 903, 268, 85, 743, 364, 74, 53, 935, 108, 42,
+ 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333,
+ 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263,
+ 44, 42, 333, 149, 519, 38, 199, 122, 39, 42,
+ 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44,
+ 39, 71, 38, 85, 359, 42, 76, 74, 85, 39,
+ 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74,
+ 44, 74, 44, 74, 53, 42, 44, 333, 39, 39,
+ 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399,
+ 229, 165, 39, 44, 327, 57, 423, 167, 39, 71,
+ 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55,
+ 58, 524, 245, 54, 50, 53, 236, 53, 81, 80,
+ 54, 50, 54, 50, 54, 50, 54, 50, 54, 50,
+ 54, 50, 54, 50, 54, 50, 85, 54, 50, 149,
+ 112, 117, 149, 49, 54, 50, 54, 50, 54, 50,
+ 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34,
+ 117, 55, 117, 54, 50, 53, 57, 53, 49, 85,
+ 333, 85, 121, 85, 841, 54, 53, 50, 56, 48,
+ 56, 837, 54, 57, 50, 57, 54, 50, 53, 54,
+ 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199,
+ 103, 87, 57, 56, 58, 87, 58, 153, 90, 98,
+ 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455,
+ 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575,
+ 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263,
+ 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71,
+ 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799,
+ 71, 39, 108, 76, 140, 135, 103, 871, 108, 44,
+ 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615,
+ 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655,
+ 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34,
+ 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149,
+ 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383,
+ 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182,
+ 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898,
+ 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236,
+ 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837,
+ 841, 229, 581, 841, 837, 41, 73, 41, 73, 137,
+ 265, 133, 37, 229, 357, 841, 837, 73, 137, 265,
+ 233, 837, 73, 137, 169, 41, 233, 837, 841, 837,
+ 841, 837, 841, 837, 841, 837, 841, 837, 841, 901,
+ 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
+ 809, 57, 805, 57, 197, 809, 57, 805, 57, 197,
+ 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71,
+ 39, 39, 327, 135, 39, 39, 39, 39, 39, 39,
+ 103, 71, 39, 39, 39, 39, 39, 39, 71, 39,
+ 135, 231, 135, 135, 39, 327, 551, 103, 167, 551,
+ 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946,
+ 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210,
+ 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266,
+ 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351,
+ 34, 3074, 7692, 63, 63,
+ };
+
+int sqlite3Fts5UnicodeCategory(int iCode) {
+ int iRes = -1;
+ int iHi;
+ int iLo;
+ int ret;
+ u16 iKey;
+
+ if( iCode>=(1<<20) ){
+ return 0;
+ }
+ iLo = aFts5UnicodeBlock[(iCode>>16)];
+ iHi = aFts5UnicodeBlock[1+(iCode>>16)];
+ iKey = (iCode & 0xFFFF);
+ while( iHi>iLo ){
+ int iTest = (iHi + iLo) / 2;
+ assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){
+ iRes = iTest;
+ iLo = iTest+1;
+ }else{
+ iHi = iTest;
+ }
+ }
+
+ if( iRes<0 ) return 0;
+ if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0;
+ ret = aFts5UnicodeData[iRes] & 0x1F;
+ if( ret!=30 ) return ret;
+ return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9;
+}
+
+void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){
+ int i = 0;
+ int iTbl = 0;
+ while( i<128 ){
+ int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ];
+ int n = (aFts5UnicodeData[iTbl] >> 5) + i;
+ for(; i<128 && ipIter;
i64 *pp = &pCsr->iInstPos;
int *po = &pCsr->iInstOff;
+ assert( sqlite3Fts5IterEof(pIter)==0 );
+ assert( pCsr->bEof==0 );
while( eDetail==FTS5_DETAIL_NONE
|| sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp)
){
pCsr->iInstPos = 0;
pCsr->iInstOff = 0;
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
if( rc==SQLITE_OK ){
rc = fts5VocabInstanceNewTerm(pCsr);
- if( eDetail==FTS5_DETAIL_NONE ) break;
+ if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break;
}
if( rc ){
pCsr->bEof = 1;
break;
}
@@ -753,12 +755,11 @@
/* xFindFunction */ 0,
/* xRename */ 0,
/* xSavepoint */ 0,
/* xRelease */ 0,
/* xRollbackTo */ 0,
+ /* xShadowName */ 0
};
void *p = (void*)pGlobal;
return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0);
}
-
-
Index: ext/fts5/fts5parse.y
==================================================================
--- ext/fts5/fts5parse.y
+++ ext/fts5/fts5parse.y
@@ -146,11 +146,15 @@
%type nearset {Fts5ExprNearset*}
%type nearphrases {Fts5ExprNearset*}
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
%destructor nearphrases { sqlite3Fts5ParseNearsetFree($$); }
-nearset(A) ::= phrase(X). { A = sqlite3Fts5ParseNearset(pParse, 0, X); }
+nearset(A) ::= phrase(Y). { A = sqlite3Fts5ParseNearset(pParse, 0, Y); }
+nearset(A) ::= CARET phrase(Y). {
+ sqlite3Fts5ParseSetCaret(Y);
+ A = sqlite3Fts5ParseNearset(pParse, 0, Y);
+}
nearset(A) ::= STRING(X) LP nearphrases(Y) neardist_opt(Z) RP. {
sqlite3Fts5ParseNear(pParse, &X);
sqlite3Fts5ParseSetDistance(pParse, Y, &Z);
A = Y;
}
@@ -187,8 +191,7 @@
/*
** Optional "*" character.
*/
%type star_opt {int}
-
star_opt(A) ::= STAR. { A = 1; }
star_opt(A) ::= . { A = 0; }
Index: ext/fts5/test/fts5aa.test
==================================================================
--- ext/fts5/test/fts5aa.test
+++ ext/fts5/test/fts5aa.test
@@ -407,10 +407,11 @@
} {200}
do_execsql_test 15.0 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 15.1 {
UPDATE t1_content SET c1 = 'xyz xyz xyz xyz xyz abc' WHERE rowid = 1;
}
do_catchsql_test 15.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
@@ -589,9 +590,21 @@
do_execsql_test 22.1 {
SELECT rowid FROM t9('a*')
} {1}
+#-------------------------------------------------------------------------
+do_execsql_test 23.0 {
+ CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
+ CREATE TABLE t11(x);
+}
+do_execsql_test 23.1 {
+ SELECT * FROM t11, t10 WHERE t11.x = t10.x AND t10.rowid IS NULL;
+}
+do_execsql_test 23.2 {
+ SELECT * FROM t11, t10 WHERE t10.rowid IS NULL;
+}
+
}
-
+expand_all_sql db
finish_test
Index: ext/fts5/test/fts5af.test
==================================================================
--- ext/fts5/test/fts5af.test
+++ ext/fts5/test/fts5af.test
@@ -172,9 +172,19 @@
);
}
do_execsql_test 5.1 {
SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
} {{[x] a a a a a...}}
+
+do_execsql_test 5.2 {
+ SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x');
+} {{[x] a a a a a}}
+do_execsql_test 5.3 {
+ SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x');
+} {{x] a a a a a...}}
+do_execsql_test 5.4 {
+ SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x');
+} {{[x a a a a a...}}
} ;# foreach_detail_mode
finish_test
ADDED ext/fts5/test/fts5cat.test
Index: ext/fts5/test/fts5cat.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5cat.test
@@ -0,0 +1,59 @@
+# 2016 Jan 15
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5cat
+
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize="unicode61 categories 'L*'");
+ INSERT INTO t1 VALUES ('Unlike1option2values3and4column5names');
+}
+
+do_execsql_test 1.1 {
+ SELECT rowid FROM t1('option');
+} {1}
+
+do_execsql_test 1.2 {
+ CREATE VIRTUAL TABLE t2 USING fts5(x);
+ CREATE VIRTUAL TABLE t2t USING fts5vocab(t2, row);
+
+ CREATE VIRTUAL TABLE t3 USING fts5(
+ x, tokenize="unicode61 categories 'L* N* Co Mn'"
+ );
+ CREATE VIRTUAL TABLE t3t USING fts5vocab(t3, row);
+
+ CREATE VIRTUAL TABLE t4 USING fts5(
+ x, tokenize="unicode61 categories 'L* N* Co M*'"
+ );
+ CREATE VIRTUAL TABLE t4t USING fts5vocab(t4, row);
+
+ INSERT INTO t2 VALUES ('สนามกีฬา');
+ INSERT INTO t3 VALUES ('สนามกีฬา');
+ INSERT INTO t4 VALUES ('สนามกีฬา');
+}
+
+do_execsql_test 1.3 {
+ SELECT * FROM t2t
+} {สนามก 1 1 ฬา 1 1}
+
+do_execsql_test 1.4 {
+ SELECT * FROM t3t
+} {สนามกีฬา 1 1}
+
+do_execsql_test 1.5 {
+ SELECT * FROM t4t
+} {สนามกีฬา 1 1}
+
+
+finish_test
Index: ext/fts5/test/fts5connect.test
==================================================================
--- ext/fts5/test/fts5connect.test
+++ ext/fts5/test/fts5connect.test
@@ -242,6 +242,5 @@
INSERT INTO ft3(ft3) VALUES('integrity-check');
}
}
finish_test
-
Index: ext/fts5/test/fts5corrupt.test
==================================================================
--- ext/fts5/test/fts5corrupt.test
+++ ext/fts5/test/fts5corrupt.test
@@ -39,19 +39,21 @@
db_save
do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
set segid [lindex [fts5_level_segids t1] 0]
+sqlite3_db_config db DEFENSIVE 0
do_test 1.3 {
execsql {
DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 4);
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {1 {database disk image is malformed}}
do_test 1.4 {
db_restore_and_reopen
+ sqlite3_db_config db DEFENSIVE 0
execsql {
UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
rowid = fts5_rowid('segment', $segid, 4);
}
catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
@@ -87,12 +89,12 @@
INSERT INTO t3 VALUES('five o');
}
do_execsql_test 3.1 {
SELECT * FROM t3 WHERE t3 MATCH 'o'
} {{one o} {three o} {five o}}
-
+sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 3.1 {
DELETE FROM t3_content WHERE rowid = 3;
SELECT * FROM t3 WHERE t3 MATCH 'o';
} {1 {database disk image is malformed}}
finish_test
Index: ext/fts5/test/fts5corrupt2.test
==================================================================
--- ext/fts5/test/fts5corrupt2.test
+++ ext/fts5/test/fts5corrupt2.test
@@ -97,10 +97,11 @@
# final leaf to some sizes may create a valid leaf page.
#
set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
set all [db eval {SELECT rowid FROM t1}]
+sqlite3_db_config db DEFENSIVE 0
for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
do_execsql_test 2.$i.1 {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE rowid=$lrowid;
}
@@ -246,10 +247,11 @@
}
}
#--------------------------------------------------------------------
reset_db
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE x5 USING fts5(tt);
INSERT INTO x5 VALUES('a');
INSERT INTO x5 VALUES('a a');
INSERT INTO x5 VALUES('a a a');
Index: ext/fts5/test/fts5corrupt3.test
==================================================================
--- ext/fts5/test/fts5corrupt3.test
+++ ext/fts5/test/fts5corrupt3.test
@@ -49,10 +49,11 @@
}]
set L [db one {SELECT length(block) FROM t1_data WHERE rowid = $rowid}]
set {} {}
} {}
+sqlite3_db_config db DEFENSIVE 0
for {set i 0} {$i < $L} {incr i} {
do_test 1.2.$i {
catchsql {
BEGIN;
UPDATE t1_data SET block = substr(block, 1, $i) WHERE id = $rowid;
@@ -84,10 +85,11 @@
#-------------------------------------------------------------------------
# Test that missing leaf pages are recognized as corruption.
#
reset_db
do_test 3.0 { create_t1 } {}
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 3.1 {
SELECT count(*) FROM t1_data;
} {105}
@@ -156,10 +158,11 @@
#-------------------------------------------------------------------------
# Test that segments that end unexpectedly are identified as corruption.
#
reset_db
+sqlite3_db_config db DEFENSIVE 0
do_test 4.0 {
execsql {
CREATE VIRTUAL TABLE t1 USING fts5(x);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
@@ -180,10 +183,11 @@
lset var end [expr $end - $i]
set struct [binary format c* $var]
db close
sqlite3 db test.db
+ sqlite3_db_config db DEFENSIVE 0
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;
}
@@ -255,10 +259,11 @@
}
#------------------------------------------------------------------------
#
reset_db
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('bbbbb ccccc');
SELECT quote(block) FROM t1_data WHERE rowid>100;
} {X'000000180630626262626201020201056363636363010203040A'}
@@ -271,10 +276,11 @@
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('aa bb cc dd ee');
SELECT pgno, quote(term) FROM t1_idx;
@@ -286,10 +292,11 @@
INSERT INTO t1(t1) VALUES('integrity-check');
} {1 {database disk image is malformed}}
#-------
reset_db
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 6.3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
INSERT INTO t1 VALUES('abc abcdef abcdefghi');
SELECT quote(block) FROM t1_data WHERE id>100;
} {X'0000001C043061626301020204036465660102030703676869010204040808'}
@@ -360,10 +367,11 @@
INSERT INTO t5 VALUES( rnddoc(10000) );
INSERT INTO t5(t5) VALUES('optimize');
}
} {}
+sqlite3_db_config db DEFENSIVE 0
do_test 7.1 {
foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] {
db eval BEGIN
db eval {DELETE FROM t5_data WHERE rowid = $i}
set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ]
@@ -381,10 +389,11 @@
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
INSERT INTO t1 VALUES('one', 'two');
}
+sqlite3_db_config db DEFENSIVE 0
do_test 9.1.1 {
set blob "12345678" ;# cookie
append blob "0105" ;# 1 level, total of 5 segments
append blob "06" ;# write counter
append blob "0002" ;# first level has 0 segments merging, 2 other.
Index: ext/fts5/test/fts5fault6.test
==================================================================
--- ext/fts5/test/fts5fault6.test
+++ ext/fts5/test/fts5fault6.test
@@ -251,11 +251,11 @@
} -body {
db eval {
SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
}
} -test {
- faultsim_test_result [list 0 $::res]
+ faultsim_test_result [list 0 $::res] {1 {SQL logic error}}
}
do_faultsim_test 5.3 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
@@ -262,11 +262,11 @@
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
}
} -test {
- faultsim_test_result {0 29}
+ faultsim_test_result {0 29} {1 {SQL logic error}}
}
do_faultsim_test 5.4 -faults oom* -prep {
faultsim_restore_and_reopen
sqlite3_fts5_create_tokenizer db tcl tcl_create
@@ -273,11 +273,11 @@
} -body {
db eval {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
}
} -test {
- faultsim_test_result {0 1}
+ faultsim_test_result {0 1} {1 {SQL logic error}}
}
#-------------------------------------------------------------------------
catch { db close }
do_faultsim_test 6 -faults oom* -prep {
Index: ext/fts5/test/fts5fault9.test
==================================================================
--- ext/fts5/test/fts5fault9.test
+++ ext/fts5/test/fts5fault9.test
@@ -22,10 +22,12 @@
return
}
foreach_detail_mode $testprefix {
+if {"%DETAIL%" != "none"} continue
+
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -96,18 +98,20 @@
do_faultsim_test 4.1 -faults oom-t* -body {
execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('2') }
} -test {
faultsim_test_result \
- {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} {1 SQLITE_NOMEM}
+ {0 {1 {0.0 0.1 0.2} 2 {0.0 0.1 0.2} 3 {0.0 0.1 0.2}}} \
+ {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
}
do_faultsim_test 4.2 -faults oom-t* -body {
execsql { SELECT rowid, fts5_test_collist(t4) FROM t4('a5 OR b5 OR c5') }
} -test {
faultsim_test_result \
- {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} {1 SQLITE_NOMEM}
+ {0 {4 {0.0 0.1 0.2} 5 {1.0 1.1 1.2} 6 {2.0 2.1 2.2}}} \
+ {1 SQLITE_NOMEM} {1 SQLITE_ERROR} {1 {SQL logic error}}
}
#-------------------------------------------------------------------------
# An OOM within an "ORDER BY rank" query.
Index: ext/fts5/test/fts5faultB.test
==================================================================
--- ext/fts5/test/fts5faultB.test
+++ ext/fts5/test/fts5faultB.test
@@ -127,8 +127,25 @@
do_faultsim_test 4.2 -faults oom* -body {
execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
} -test {
faultsim_test_result {0 {2 3}}
}
+
+#-------------------------------------------------------------------------
+# Test OOM injection while parsing a CARET expression
+#
+reset_db
+do_execsql_test 5.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a);
+ INSERT INTO t1 VALUES('a b c d'); -- 1
+ INSERT INTO t1 VALUES('d a b c'); -- 2
+ INSERT INTO t1 VALUES('c d a b'); -- 3
+ INSERT INTO t1 VALUES('b c d a'); -- 4
+}
+do_faultsim_test 5.1 -faults oom* -body {
+ execsql { SELECT rowid FROM t1('^a OR ^b') }
+} -test {
+ faultsim_test_result {0 {1 4}}
+}
finish_test
ADDED ext/fts5/test/fts5first.test
Index: ext/fts5/test/fts5first.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5first.test
@@ -0,0 +1,95 @@
+# 2017 November 25
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5first
+
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+}
+
+foreach {tn expr ok} {
+ 1 {^abc} 1
+ 2 {^abc + def} 1
+ 3 {^ "abc def"} 1
+ 4 {^"abc def"} 1
+ 5 {abc ^def} 1
+ 6 {abc + ^def} 0
+ 7 {abc ^+ def} 0
+ 8 {"^abc"} 1
+ 9 {NEAR(^abc def)} 0
+} {
+ set res(0) {/1 {fts5: syntax error near .*}/}
+ set res(1) {0 {}}
+
+ do_catchsql_test 1.$tn { SELECT * FROM x1($expr) } $res($ok)
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 2.0 {
+ INSERT INTO x1 VALUES('a b c', 'b c a');
+}
+
+foreach {tn expr match} {
+ 1 {^a} 1
+ 2 {^b} 1
+ 3 {^c} 0
+ 4 {^a + b} 1
+ 5 {^b + c} 1
+ 6 {^c + a} 0
+ 7 {^"c a"} 0
+ 8 {a:^a} 1
+ 9 {a:^b} 0
+ 10 {a:^"a b"} 1
+} {
+ do_execsql_test 2.$tn { SELECT EXISTS (SELECT rowid FROM x1($expr)) } $match
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.0 {
+ DELETE FROM x1;
+ INSERT INTO x1 VALUES('b a', 'c a');
+ INSERT INTO x1 VALUES('a a', 'c c');
+ INSERT INTO x1 VALUES('a b', 'a a');
+}
+fts5_aux_test_functions db
+
+foreach {tn expr expect} {
+ 1 {^a} {{2 1}}
+ 2 {^c AND ^b} {{0 2} {1 0}}
+} {
+ do_execsql_test 3.$tn {
+ SELECT fts5_test_queryphrase(x1) FROM x1($expr) LIMIT 1
+ } [list $expect]
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 3.1 {
+ CREATE VIRTUAL TABLE x2 USING fts5(a, b, c, detail=column);
+}
+
+do_catchsql_test 3.2 {
+ SELECT * FROM x2('a + b');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+
+do_catchsql_test 3.3 {
+ SELECT * FROM x2('^a');
+} {1 {fts5: phrase queries are not supported (detail!=full)}}
+finish_test
Index: ext/fts5/test/fts5integrity.test
==================================================================
--- ext/fts5/test/fts5integrity.test
+++ ext/fts5/test/fts5integrity.test
@@ -69,10 +69,11 @@
} {1 1 1 1 1}
do_execsql_test 4.1 {
INSERT INTO aa(aa) VALUES('integrity-check');
}
+sqlite3_db_config db DEFENSIVE 0
do_catchsql_test 4.2 {
BEGIN;
UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
INSERT INTO aa(aa) VALUES('integrity-check');
} {1 {database disk image is malformed}}
Index: ext/fts5/test/fts5plan.test
==================================================================
--- ext/fts5/test/fts5plan.test
+++ ext/fts5/test/fts5plan.test
@@ -27,40 +27,39 @@
}
do_eqp_test 1.1 {
SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
} {
- 0 0 0 {SCAN TABLE t1}
- 0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
+ QUERY PLAN
+ |--SCAN TABLE t1
+ `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
}
do_eqp_test 1.2 {
SELECT * FROM t1, f1 WHERE f1 > t1.x
} {
- 0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
- 0 1 0 {SCAN TABLE t1}
+ QUERY PLAN
+ |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+ `--SCAN TABLE t1
}
do_eqp_test 1.3 {
SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
} {
- 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:}
- 0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+ QUERY PLAN
+ |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
+ `--USE TEMP B-TREE FOR ORDER BY
}
do_eqp_test 1.4 {
SELECT * FROM f1 ORDER BY rank
} {
- 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:}
- 0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+ QUERY PLAN
+ |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
+ `--USE TEMP B-TREE FOR ORDER BY
}
do_eqp_test 1.5 {
SELECT * FROM f1 WHERE rank MATCH ?
-} {
- 0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
-}
-
-
-
+} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
finish_test
Index: ext/fts5/test/fts5query.test
==================================================================
--- ext/fts5/test/fts5query.test
+++ ext/fts5/test/fts5query.test
@@ -62,11 +62,11 @@
execsql { INSERT INTO t1 VALUES($doc) }
}
execsql COMMIT
} {}
- do_execsql_test 1.$tn.2 {
+ do_execsql_test 2.$tn.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
set ret 1
foreach x [list a c e g i k m o q s u] {
@@ -75,7 +75,17 @@
} {}
incr ret
}
}
+reset_db
+do_execsql_test 3.0 {
+ CREATE VIRTUAL TABLE x1 USING fts5(a);
+ INSERT INTO x1(rowid, a) VALUES(-1000000000000, 'toyota');
+ INSERT INTO x1(rowid, a) VALUES(1, 'tarago');
+}
+do_execsql_test 3.1 {
+ SELECT rowid FROM x1('t*');
+} {-1000000000000 1}
+
finish_test
Index: ext/fts5/test/fts5rank.test
==================================================================
--- ext/fts5/test/fts5rank.test
+++ ext/fts5/test/fts5rank.test
@@ -146,9 +146,20 @@
SELECT * FROM VTest WHERE
VTest MATCH 'wrinkle in time OR a wrinkle in time' ORDER BY rank;
} {{wrinkle in time} {Bill Smith}}
-
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+ CREATE VIRTUAL TABLE ttt USING fts5(a);
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+ )
+ INSERT INTO ttt SELECT 'word ' || i FROM s;
+}
+do_execsql_test 5.1 {
+ SELECT rowid FROM ttt('word') WHERE rowid BETWEEN 30 AND 40 ORDER BY rank;
+} {30 31 32 33 34 35 36 37 38 39 40}
finish_test
Index: ext/fts5/test/fts5rebuild.test
==================================================================
--- ext/fts5/test/fts5rebuild.test
+++ ext/fts5/test/fts5rebuild.test
@@ -37,10 +37,11 @@
do_execsql_test 1.4 {
INSERT INTO f1(f1) VALUES('integrity-check');
} {}
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.5 {
DELETE FROM f1_data;
} {}
do_catchsql_test 1.6 {
Index: ext/fts5/test/fts5rowid.test
==================================================================
--- ext/fts5/test/fts5rowid.test
+++ ext/fts5/test/fts5rowid.test
@@ -68,10 +68,11 @@
set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.3 {
SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 2.4 {
UPDATE x1_data SET block = X'';
SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
Index: ext/fts5/test/fts5unicode.test
==================================================================
--- ext/fts5/test/fts5unicode.test
+++ ext/fts5/test/fts5unicode.test
@@ -39,11 +39,10 @@
}
#-------------------------------------------------------------------------
# Check that "unicode61" really is the default tokenizer.
#
-
do_execsql_test 2.0 "
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t2 USING fts5(x, tokenize = unicode61);
CREATE VIRTUAL TABLE t3 USING fts5(x, tokenize = ascii);
INSERT INTO t1 VALUES('\xC0\xC8\xCC');
@@ -53,8 +52,34 @@
do_execsql_test 2.1 "
SELECT 't1' FROM t1 WHERE t1 MATCH '\xE0\xE8\xEC';
SELECT 't2' FROM t2 WHERE t2 MATCH '\xE0\xE8\xEC';
SELECT 't3' FROM t3 WHERE t3 MATCH '\xE0\xE8\xEC';
" {t1 t2}
+
+#-------------------------------------------------------------------------
+# Check that codepoints that require 4 bytes to store in utf-8 (those that
+# require 17 or more bits to store).
+#
+
+set A [db one {SELECT char(0x1F75E)}] ;# Type So
+set B [db one {SELECT char(0x1F5FD)}] ;# Type So
+set C [db one {SELECT char(0x2F802)}] ;# Type Lo
+set D [db one {SELECT char(0x2F808)}] ;# Type Lo
+
+do_execsql_test 3.0 "
+ CREATE VIRTUAL TABLE xyz USING fts5(x,
+ tokenize = \"unicode61 separators '$C' tokenchars '$A'\"
+ );
+ CREATE VIRTUAL TABLE xyz_v USING fts5vocab(xyz, row);
+
+ INSERT INTO xyz VALUES('$A$B$C$D');
+"
+
+do_execsql_test 3.1 {
+ SELECT * FROM xyz_v;
+} [list $A 1 1 $D 1 1]
+
+
+
finish_test
ADDED ext/fts5/test/fts5unicode4.test
Index: ext/fts5/test/fts5unicode4.test
==================================================================
--- /dev/null
+++ ext/fts5/test/fts5unicode4.test
@@ -0,0 +1,31 @@
+# 2018 July 25
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5unicode4
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE sss USING fts5(a, prefix=3);
+}
+
+do_execsql_test 1.1 {
+ INSERT INTO sss VALUES('まりや');
+}
+
+finish_test
Index: ext/fts5/test/fts5version.test
==================================================================
--- ext/fts5/test/fts5version.test
+++ ext/fts5/test/fts5version.test
@@ -34,13 +34,14 @@
do_execsql_test 1.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'a';
} {1}
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 1.4 {
UPDATE t1_config set v=5 WHERE k='version';
-}
+}
do_test 1.5 {
db close
sqlite3 db test.db
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
@@ -51,13 +52,14 @@
sqlite3 db test.db
catchsql { INSERT INTO t1 VALUES('x y z') }
} {1 {invalid fts5 file format (found 5, expected 4) - run 'rebuild'}}
do_test 1.7 {
+ sqlite3_db_config db DEFENSIVE 0
execsql { DELETE FROM t1_config WHERE k='version' }
db close
sqlite3 db test.db
catchsql { SELECT * FROM t1 WHERE t1 MATCH 'a' }
} {1 {invalid fts5 file format (found 0, expected 4) - run 'rebuild'}}
finish_test
Index: ext/fts5/test/fts5vocab.test
==================================================================
--- ext/fts5/test/fts5vocab.test
+++ ext/fts5/test/fts5vocab.test
@@ -418,10 +418,11 @@
if {[detail_is_none]} { set resc [row_to_col $resr] }
do_execsql_test 8.1.1 { SELECT * FROM x1_r; } $resr
do_execsql_test 8.1.2 { SELECT * FROM x1_c } $resc
+sqlite3_db_config db DEFENSIVE 0
do_execsql_test 8.2 {
PRAGMA writable_schema = 1;
UPDATE sqlite_master
SET sql = 'CREATE VIRTUAL TABLE x1 USING fts5(a, detail=%DETAIL%)'
WHERE name = 'x1';
@@ -479,6 +480,5 @@
} 0
finish_test
-
Index: ext/fts5/test/fts5vocab2.test
==================================================================
--- ext/fts5/test/fts5vocab2.test
+++ ext/fts5/test/fts5vocab2.test
@@ -11,11 +11,11 @@
#
# The tests in this file focus on testing the fts5vocab module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5vocab
+set testprefix fts5vocab2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
@@ -204,6 +204,5 @@
SELECT * FROM v1;
} {
}
finish_test
-
Index: ext/icu/README.txt
==================================================================
--- ext/icu/README.txt
+++ ext/icu/README.txt
@@ -37,12 +37,12 @@
http://www.icu-project.org/userguide/posix.html#case_mappings
To utilise "general" case mapping, the upper() or lower() scalar
functions are invoked with one argument:
- upper('ABC') -> 'abc'
- lower('abc') -> 'ABC'
+ upper('abc') -> 'ABC'
+ lower('ABC') -> 'abc'
To access ICU "language specific" case mapping, upper() or lower()
should be invoked with two arguments. The second argument is the name
of the locale to use. Passing an empty string ("") or SQL NULL value
as the second argument is the same as invoking the 1 argument version
Index: ext/icu/icu.c
==================================================================
--- ext/icu/icu.c
+++ ext/icu/icu.c
@@ -26,11 +26,13 @@
**
** * An implementation of the LIKE operator that uses ICU to
** provide case-independent matching.
*/
-#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+#if !defined(SQLITE_CORE) \
+ || defined(SQLITE_ENABLE_ICU) \
+ || defined(SQLITE_ENABLE_ICU_COLLATIONS)
/* Include ICU headers */
#include
#include
#include
@@ -43,10 +45,30 @@
SQLITE_EXTENSION_INIT1
#else
#include "sqlite3.h"
#endif
+/*
+** This function is called when an ICU function called from within
+** the implementation of an SQL scalar function returns an error.
+**
+** The scalar function context passed as the first argument is
+** loaded with an error message based on the following two args.
+*/
+static void icuFunctionError(
+ sqlite3_context *pCtx, /* SQLite scalar function context */
+ const char *zName, /* Name of ICU function that failed */
+ UErrorCode e /* Error code returned by ICU function */
+){
+ char zBuf[128];
+ sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
+ zBuf[127] = '\0';
+ sqlite3_result_error(pCtx, zBuf, -1);
+}
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
+
/*
** Maximum length (in bytes) of the pattern in a LIKE or GLOB
** operator.
*/
#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH
@@ -222,28 +244,10 @@
if( zA && zB ){
sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc));
}
}
-/*
-** This function is called when an ICU function called from within
-** the implementation of an SQL scalar function returns an error.
-**
-** The scalar function context passed as the first argument is
-** loaded with an error message based on the following two args.
-*/
-static void icuFunctionError(
- sqlite3_context *pCtx, /* SQLite scalar function context */
- const char *zName, /* Name of ICU function that failed */
- UErrorCode e /* Error code returned by ICU function */
-){
- char zBuf[128];
- sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e));
- zBuf[127] = '\0';
- sqlite3_result_error(pCtx, zBuf, -1);
-}
-
/*
** Function to delete compiled regexp objects. Registered as
** a destructor function with sqlite3_set_auxdata().
*/
static void icuRegexpDelete(void *p){
@@ -405,10 +409,12 @@
return;
}
assert( 0 ); /* Unreachable */
}
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
+
/*
** Collation sequence destructor function. The pCtx argument points to
** a UCollator structure previously allocated using ucol_open().
*/
static void icuCollationDel(void *pCtx){
@@ -499,10 +505,11 @@
unsigned short enc; /* Optimal text encoding */
unsigned char iContext; /* sqlite3_user_data() context */
void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
} scalars[] = {
{"icu_load_collation", 2, SQLITE_UTF8, 1, icuLoadCollation},
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU)
{"regexp", 2, SQLITE_ANY|SQLITE_DETERMINISTIC, 0, icuRegexpFunc},
{"lower", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"lower", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF16|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
@@ -510,14 +517,14 @@
{"lower", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuCaseFunc16},
{"upper", 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"upper", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 1, icuCaseFunc16},
{"like", 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
{"like", 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, icuLikeFunc},
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */
};
int rc = SQLITE_OK;
int i;
-
for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){
const struct IcuScalar *p = &scalars[i];
rc = sqlite3_create_function(
db, p->zName, p->nArg, p->enc,
Index: ext/lsm1/Makefile
==================================================================
--- ext/lsm1/Makefile
+++ ext/lsm1/Makefile
@@ -41,16 +41,16 @@
$(LSMDIR)/lsm-test/lsmtest_util.c $(LSMDIR)/lsm-test/lsmtest_win32.c
# all: lsm.so
-LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR)
+LSMOPTS += -DLSM_MUTEX_PTHREADS=1 -I$(LSMDIR) -DHAVE_ZLIB
lsm.so: $(LSMOBJ)
$(TCCX) -shared -o lsm.so $(LSMOBJ)
%.o: $(LSMDIR)/%.c $(LSMHDR) sqlite3.h
$(TCCX) $(LSMOPTS) -c $<
lsmtest$(EXE): $(LSMOBJ) $(LSMTESTSRC) $(LSMTESTHDR) sqlite3.o
# $(TCPPX) -c $(TOP)/lsm-test/lsmtest_tdb2.cc
- $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB)
+ $(TCCX) $(LSMOPTS) $(LSMTESTSRC) $(LSMOBJ) sqlite3.o -o lsmtest$(EXE) $(THREADLIB) -lz
Index: ext/lsm1/lsm-test/lsmtest.h
==================================================================
--- ext/lsm1/lsm-test/lsmtest.h
+++ ext/lsm1/lsm-test/lsmtest.h
@@ -119,10 +119,11 @@
** includes code for a couple of different lsm configurations, and for
** various types of fault injection and robustness testing.
*/
int test_lsm_open(const char*, const char *zFile, int bClear, TestDb **ppDb);
int test_lsm_lomem_open(const char*, const char*, int bClear, TestDb **ppDb);
+int test_lsm_lomem2_open(const char*, const char*, int bClear, TestDb **ppDb);
int test_lsm_zip_open(const char*, const char*, int bClear, TestDb **ppDb);
int test_lsm_small_open(const char*, const char*, int bClear, TestDb **ppDb);
int test_lsm_mt2(const char*, const char *zFile, int bClear, TestDb **ppDb);
int test_lsm_mt3(const char*, const char *zFile, int bClear, TestDb **ppDb);
Index: ext/lsm1/lsm-test/lsmtest1.c
==================================================================
--- ext/lsm1/lsm-test/lsmtest1.c
+++ ext/lsm1/lsm-test/lsmtest1.c
@@ -272,10 +272,11 @@
int i;
int iDot;
int rc = LSM_OK;
Datasource *pData;
TestDb *pDb;
+ int iToggle = 0;
/* Start the test case, open a database and allocate the datasource. */
pDb = testOpen(zSystem, 1, &rc);
pData = testDatasourceNew(&p->defn);
@@ -285,12 +286,15 @@
/* Insert some data */
testWriteDatasourceRange(pDb, pData, i, p->nVerify, &rc);
i += p->nVerify;
+ if( iToggle ) testBegin(pDb, 1, &rc);
/* Check that the db content is correct. */
testDbContents(pDb, pData, p->nRow, 0, i-1, p->nTest, p->bTestScan, &rc);
+ if( iToggle ) testCommit(pDb, 0, &rc);
+ iToggle = (iToggle+1)%2;
if( bRecover ){
testReopenRecover(&pDb, &rc);
}else{
testReopen(&pDb, &rc);
Index: ext/lsm1/lsm-test/lsmtest_tdb.c
==================================================================
--- ext/lsm1/lsm-test/lsmtest_tdb.c
+++ ext/lsm1/lsm-test/lsmtest_tdb.c
@@ -719,10 +719,11 @@
int (*xOpen)(const char *, const char *zFilename, int bClear, TestDb **ppDb);
} aLib[] = {
{ "sqlite3", "testdb.sqlite", sql_open },
{ "lsm_small", "testdb.lsm_small", test_lsm_small_open },
{ "lsm_lomem", "testdb.lsm_lomem", test_lsm_lomem_open },
+ { "lsm_lomem2", "testdb.lsm_lomem2", test_lsm_lomem2_open },
#ifdef HAVE_ZLIB
{ "lsm_zip", "testdb.lsm_zip", test_lsm_zip_open },
#endif
{ "lsm", "testdb.lsm", test_lsm_open },
#ifdef LSM_MUTEX_PTHREADS
Index: ext/lsm1/lsm-test/lsmtest_tdb3.c
==================================================================
--- ext/lsm1/lsm-test/lsmtest_tdb3.c
+++ ext/lsm1/lsm-test/lsmtest_tdb3.c
@@ -615,12 +615,16 @@
LsmDb *pDb = (LsmDb *)pTestDb;
lsm_cursor *csr;
if( pKey==0 ) return LSM_OK;
- rc = lsm_csr_open(pDb->db, &csr);
- if( rc!=LSM_OK ) return rc;
+ if( pDb->pCsr==0 ){
+ rc = lsm_csr_open(pDb->db, &csr);
+ if( rc!=LSM_OK ) return rc;
+ }else{
+ csr = pDb->pCsr;
+ }
rc = lsm_csr_seek(csr, pKey, nKey, LSM_SEEK_EQ);
if( rc==LSM_OK ){
if( lsm_csr_valid(csr) ){
const void *pVal; int nVal;
@@ -636,11 +640,13 @@
}else{
*ppVal = 0;
*pnVal = -1;
}
}
- lsm_csr_close(csr);
+ if( pDb->pCsr==0 ){
+ lsm_csr_close(csr);
+ }
return rc;
}
static int test_lsm_scan(
TestDb *pTestDb,
@@ -650,14 +656,32 @@
void *pLast, int nLast,
void (*xCallback)(void *, void *, int , void *, int)
){
LsmDb *pDb = (LsmDb *)pTestDb;
lsm_cursor *csr;
+ lsm_cursor *csr2 = 0;
int rc;
- rc = lsm_csr_open(pDb->db, &csr);
- if( rc!=LSM_OK ) return rc;
+ if( pDb->pCsr==0 ){
+ rc = lsm_csr_open(pDb->db, &csr);
+ if( rc!=LSM_OK ) return rc;
+ }else{
+ rc = LSM_OK;
+ csr = pDb->pCsr;
+ }
+
+ /* To enhance testing, if both pLast and pFirst are defined, seek the
+ ** cursor to the "end" boundary here. Then the next block seeks it to
+ ** the "start" ready for the scan. The point is to test that cursors
+ ** can be reused. */
+ if( pLast && pFirst ){
+ if( bReverse ){
+ rc = lsm_csr_seek(csr, pFirst, nFirst, LSM_SEEK_LE);
+ }else{
+ rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_GE);
+ }
+ }
if( bReverse ){
if( pLast ){
rc = lsm_csr_seek(csr, pLast, nLast, LSM_SEEK_LE);
}else{
@@ -694,11 +718,13 @@
}else{
rc = lsm_csr_next(csr);
}
}
- lsm_csr_close(csr);
+ if( pDb->pCsr==0 ){
+ lsm_csr_close(csr);
+ }
return rc;
}
static int test_lsm_begin(TestDb *pTestDb, int iLevel){
int rc = LSM_OK;
@@ -759,10 +785,11 @@
#define TEST_COMPRESSION -3
#define TEST_MT_MODE -2
#define TEST_MT_MIN_CKPT -4
#define TEST_MT_MAX_CKPT -5
+
int test_lsm_config_str(
LsmDb *pLsm,
lsm_db *db,
int bWorker,
@@ -1030,10 +1057,23 @@
"autocheckpoint=32"
"mmap=0 "
;
return testLsmOpen(zCfg, zFilename, bClear, ppDb);
}
+
+int test_lsm_lomem2_open(
+ const char *zSpec,
+ const char *zFilename,
+ int bClear,
+ TestDb **ppDb
+){
+ /* "max_freelist=4 autocheckpoint=32" */
+ const char *zCfg =
+ "page_size=512 block_size=64 autoflush=0 mmap=0 "
+ ;
+ return testLsmOpen(zCfg, zFilename, bClear, ppDb);
+}
int test_lsm_zip_open(
const char *zSpec,
const char *zFilename,
int bClear,
Index: ext/lsm1/lsmInt.h
==================================================================
--- ext/lsm1/lsmInt.h
+++ ext/lsm1/lsmInt.h
@@ -108,11 +108,11 @@
typedef lsm_i64 i64;
typedef unsigned long long int u64;
#endif
/* A page number is a 64-bit integer. */
-typedef i64 Pgno;
+typedef i64 LsmPgno;
#ifdef LSM_DEBUG
int lsmErrorBkpt(int);
#else
# define lsmErrorBkpt(x) (x)
@@ -400,13 +400,13 @@
TreeHeader treehdr; /* Local copy of tree-header */
u32 aSnapshot[LSM_META_PAGE_SIZE / sizeof(u32)];
};
struct Segment {
- Pgno iFirst; /* First page of this run */
- Pgno iLastPg; /* Last page of this run */
- Pgno iRoot; /* Root page number (if any) */
+ LsmPgno iFirst; /* First page of this run */
+ LsmPgno iLastPg; /* Last page of this run */
+ LsmPgno iRoot; /* Root page number (if any) */
int nSize; /* Size of this run in pages */
Redirect *pRedirect; /* Block redirects (or NULL) */
};
@@ -454,20 +454,20 @@
** iOutputOff:
** The byte offset to write to next within the last page of the
** output segment.
*/
struct MergeInput {
- Pgno iPg; /* Page on which next input is stored */
+ LsmPgno iPg; /* Page on which next input is stored */
int iCell; /* Cell containing next input to merge */
};
struct Merge {
int nInput; /* Number of input runs being merged */
MergeInput *aInput; /* Array nInput entries in size */
MergeInput splitkey; /* Location in file of current splitkey */
int nSkip; /* Number of separators entries to skip */
int iOutputOff; /* Write offset on output page */
- Pgno iCurrentPtr; /* Current pointer value */
+ LsmPgno iCurrentPtr; /* Current pointer value */
};
/*
** The first argument to this macro is a pointer to a Segment structure.
** Returns true if the structure instance indicates that the separators
@@ -577,14 +577,14 @@
i64 iId; /* Snapshot id */
i64 iLogOff; /* Log file offset */
Redirect redirect; /* Block redirection array */
/* Used by worker snapshots only */
- int nBlock; /* Number of blocks in database file */
- Pgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */
- Freelist freelist; /* Free block list */
- u32 nWrite; /* Total number of pages written to disk */
+ int nBlock; /* Number of blocks in database file */
+ LsmPgno aiAppend[LSM_APPLIST_SZ]; /* Append point list */
+ Freelist freelist; /* Free block list */
+ u32 nWrite; /* Total number of pages written to disk */
};
#define LSM_INITIAL_SNAPSHOT_ID 11
/*
** Functions from file "lsm_ckpt.c".
@@ -708,11 +708,11 @@
void lsmFsSetPageSize(FileSystem *, int);
int lsmFsFileid(lsm_db *pDb, void **ppId, int *pnId);
/* Creating, populating, gobbling and deleting sorted runs. */
-void lsmFsGobble(lsm_db *, Segment *, Pgno *, int);
+void lsmFsGobble(lsm_db *, Segment *, LsmPgno *, int);
int lsmFsSortedDelete(FileSystem *, Snapshot *, int, Segment *);
int lsmFsSortedFinish(FileSystem *, Segment *);
int lsmFsSortedAppend(FileSystem *, Snapshot *, Level *, int, Page **);
int lsmFsSortedPadding(FileSystem *, Snapshot *, Segment *);
@@ -725,18 +725,18 @@
void lsmSortedSplitkey(lsm_db *, Level *, int *);
/* Reading sorted run content. */
int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg);
-int lsmFsDbPageGet(FileSystem *, Segment *, Pgno, Page **);
+int lsmFsDbPageGet(FileSystem *, Segment *, LsmPgno, Page **);
int lsmFsDbPageNext(Segment *, Page *, int eDir, Page **);
u8 *lsmFsPageData(Page *, int *);
int lsmFsPageRelease(Page *);
int lsmFsPagePersist(Page *);
void lsmFsPageRef(Page *);
-Pgno lsmFsPageNumber(Page *);
+LsmPgno lsmFsPageNumber(Page *);
int lsmFsNRead(FileSystem *);
int lsmFsNWrite(FileSystem *);
int lsmFsMetaPageGet(FileSystem *, int, int, MetaPage **);
@@ -746,11 +746,11 @@
#ifdef LSM_DEBUG
int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg);
int lsmFsIntegrityCheck(lsm_db *);
#endif
-Pgno lsmFsRedirectPage(FileSystem *, Redirect *, Pgno);
+LsmPgno lsmFsRedirectPage(FileSystem *, Redirect *, LsmPgno);
int lsmFsPageWritable(Page *);
/* Functions to read, write and sync the log file. */
int lsmFsWriteLog(FileSystem *pFS, i64 iOff, LsmString *pStr);
@@ -766,12 +766,12 @@
int lsmFsSyncDb(FileSystem *, int);
void lsmFsFlushWaiting(FileSystem *, int *);
/* Used by lsm_info(ARRAY_STRUCTURE) and lsm_config(MMAP) */
-int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, Pgno iFirst, char **pzOut);
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut);
+int lsmInfoArrayStructure(lsm_db *pDb, int bBlock, LsmPgno iFirst, char **pz);
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut);
int lsmConfigMmap(lsm_db *pDb, int *piParam);
int lsmEnvOpen(lsm_env *, const char *, int, lsm_file **);
int lsmEnvClose(lsm_env *pEnv, lsm_file *pFile);
int lsmEnvLock(lsm_env *pEnv, lsm_file *pFile, int iLock, int eLock);
@@ -783,11 +783,11 @@
void lsmEnvSleep(lsm_env *, int);
int lsmFsReadSyncedId(lsm_db *db, int, i64 *piVal);
-int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, Pgno, int *);
+int lsmFsSegmentContainsPg(FileSystem *pFS, Segment *, LsmPgno, int *);
void lsmFsPurgeCache(FileSystem *);
/*
** End of functions from "lsm_file.c".
@@ -794,11 +794,11 @@
**************************************************************************/
/*
** Functions from file "lsm_sorted.c".
*/
-int lsmInfoPageDump(lsm_db *, Pgno, int, char **);
+int lsmInfoPageDump(lsm_db *, LsmPgno, int, char **);
void lsmSortedCleanup(lsm_db *);
int lsmSortedAutoWork(lsm_db *, int nUnit);
int lsmSortedWalkFreelist(lsm_db *, int, int (*)(void *, int, i64), void *);
Index: ext/lsm1/lsm_ckpt.c
==================================================================
--- ext/lsm1/lsm_ckpt.c
+++ ext/lsm1/lsm_ckpt.c
@@ -387,11 +387,11 @@
CkptBuffer *p, /* Checkpoint buffer to write to */
int *piOut, /* IN/OUT: Offset within checkpoint buffer */
int *pRc /* IN/OUT: Error code */
){
int i;
- Pgno *aiAppend = db->pWorker->aiAppend;
+ LsmPgno *aiAppend = db->pWorker->aiAppend;
for(i=0; inPagesize <= pFS->nMapLimit);
}
/*
** Given that there are currently nHash slots in the hash table, return
** the hash key for file iFile, page iPg.
*/
-static int fsHashKey(int nHash, Pgno iPg){
+static int fsHashKey(int nHash, LsmPgno iPg){
return (iPg % nHash);
}
/*
** This is a helper function for lsmFsOpen(). It opens a single file on
@@ -878,17 +878,17 @@
**
** For a compressed database, page numbers are byte offsets. The first
** page on each block is the byte offset immediately following the 4-byte
** "previous block" pointer at the start of each block.
*/
-static Pgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
- Pgno iPg;
+static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
+ LsmPgno iPg;
if( pFS->pCompress ){
if( iBlock==1 ){
iPg = pFS->nMetasize * 2 + 4;
}else{
- iPg = pFS->nBlocksize * (Pgno)(iBlock-1) + 4;
+ iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
}
}else{
const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
if( iBlock==1 ){
iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize);
@@ -905,13 +905,13 @@
**
** For a compressed database, page numbers are byte offsets. The first
** page on each block is the byte offset of the byte immediately before
** the 4-byte "next block" pointer at the end of each block.
*/
-static Pgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
+static LsmPgno fsLastPageOnBlock(FileSystem *pFS, int iBlock){
if( pFS->pCompress ){
- return pFS->nBlocksize * (Pgno)iBlock - 1 - 4;
+ return pFS->nBlocksize * (LsmPgno)iBlock - 1 - 4;
}else{
const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
return iBlock * nPagePerBlock;
}
}
@@ -918,11 +918,11 @@
/*
** Return the block number of the block that page iPg is located on.
** Blocks are numbered starting from 1.
*/
-static int fsPageToBlock(FileSystem *pFS, Pgno iPg){
+static int fsPageToBlock(FileSystem *pFS, LsmPgno iPg){
if( pFS->pCompress ){
return (int)((iPg / pFS->nBlocksize) + 1);
}else{
return (int)(1 + ((iPg-1) / (pFS->nBlocksize / pFS->nPagesize)));
}
@@ -931,11 +931,11 @@
/*
** Return true if page iPg is the last page on its block.
**
** This function is only called in non-compressed database mode.
*/
-static int fsIsLast(FileSystem *pFS, Pgno iPg){
+static int fsIsLast(FileSystem *pFS, LsmPgno iPg){
const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
assert( !pFS->pCompress );
return ( iPg && (iPg % nPagePerBlock)==0 );
}
@@ -942,11 +942,11 @@
/*
** Return true if page iPg is the first page on its block.
**
** This function is only called in non-compressed database mode.
*/
-static int fsIsFirst(FileSystem *pFS, Pgno iPg){
+static int fsIsFirst(FileSystem *pFS, LsmPgno iPg){
const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
assert( !pFS->pCompress );
return ( (iPg % nPagePerBlock)==1
|| (iPgflags & PAGE_DIRTY)==0 ); */
return pPage ? pPage->iPg : 0;
}
/*
@@ -1056,11 +1056,11 @@
** to it. Otherwise, return NULL.
**
** Either way, if argument piHash is not NULL set *piHash to the hash slot
** number that page iPg would be stored in before returning.
*/
-static Page *fsPageFindInHash(FileSystem *pFS, Pgno iPg, int *piHash){
+static Page *fsPageFindInHash(FileSystem *pFS, LsmPgno iPg, int *piHash){
Page *p; /* Return value */
int iHash = fsHashKey(pFS->nHash, iPg);
if( piHash ) *piHash = iHash;
for(p=pFS->apHash[iHash]; p; p=p->pHashNext){
@@ -1187,12 +1187,12 @@
/*
** If page iPg has been redirected according to the redirections in the
** object passed as the second argument, return the destination page to
** which it is redirected. Otherwise, return a copy of iPg.
*/
-Pgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, Pgno iPg){
- Pgno iReal = iPg;
+LsmPgno lsmFsRedirectPage(FileSystem *pFS, Redirect *pRedir, LsmPgno iPg){
+ LsmPgno iReal = iPg;
if( pRedir ){
const int nPagePerBlock = (
pFS->pCompress ? pFS->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
);
@@ -1201,11 +1201,11 @@
for(i=0; in; i++){
int iFrom = pRedir->a[i].iFrom;
if( iFrom>iBlk ) break;
if( iFrom==iBlk ){
int iTo = pRedir->a[i].iTo;
- iReal = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+ iReal = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
if( iTo==1 ){
iReal += (fsFirstPageOnBlock(pFS, 1)-1);
}
break;
}
@@ -1215,11 +1215,11 @@
assert( iReal!=0 );
return iReal;
}
/* Required by the circular fsBlockNext<->fsPageGet dependency. */
-static int fsPageGet(FileSystem *, Segment *, Pgno, int, Page **, int *);
+static int fsPageGet(FileSystem *, Segment *, LsmPgno, int, Page **, int *);
/*
** Parameter iBlock is a database file block. This function reads the value
** stored in the blocks "next block" pointer and stores it in *piNext.
** LSM_OK is returned if everything is successful, or an LSM error code
@@ -1267,11 +1267,11 @@
}
/*
** Return the page number of the last page on the same block as page iPg.
*/
-Pgno fsLastPageOnPagesBlock(FileSystem *pFS, Pgno iPg){
+LsmPgno fsLastPageOnPagesBlock(FileSystem *pFS, LsmPgno iPg){
return fsLastPageOnBlock(pFS, fsPageToBlock(pFS, iPg));
}
/*
** Read nData bytes of data from offset iOff of the database file into
@@ -1535,11 +1535,11 @@
** If no error occurs, LSM_OK is returned. Otherwise, an lsm error code.
*/
static int fsPageGet(
FileSystem *pFS, /* File-system handle */
Segment *pSeg, /* Block redirection to use (or NULL) */
- Pgno iPg, /* Page id */
+ LsmPgno iPg, /* Page id */
int noContent, /* True to not load content from disk */
Page **ppPg, /* OUT: New page handle */
int *pnSpace /* OUT: Bytes of free space */
){
Page *p;
@@ -1547,11 +1547,11 @@
int rc = LSM_OK;
/* In most cases iReal is the same as iPg. Except, if pSeg->pRedirect is
** not NULL, and the block containing iPg has been redirected, then iReal
** is the page number after redirection. */
- Pgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
+ LsmPgno iReal = lsmFsRedirectPage(pFS, (pSeg ? pSeg->pRedirect : 0), iPg);
assert_lists_are_ok(pFS);
assert( iPg>=fsFirstPageOnBlock(pFS, 1) );
assert( iReal>=fsFirstPageOnBlock(pFS, 1) );
*ppPg = 0;
@@ -1687,12 +1687,12 @@
** and iLast, inclusive, and pRun is not equal to pIgnore.
*/
static int fsRunEndsBetween(
Segment *pRun,
Segment *pIgnore,
- Pgno iFirst,
- Pgno iLast
+ LsmPgno iFirst,
+ LsmPgno iLast
){
return (pRun!=pIgnore && (
(pRun->iFirst>=iFirst && pRun->iFirst<=iLast)
|| (pRun->iLastPg>=iFirst && pRun->iLastPg<=iLast)
));
@@ -1703,12 +1703,12 @@
** which the first or last page is between iFirst and iLast, inclusive.
*/
static int fsLevelEndsBetween(
Level *pLevel,
Segment *pIgnore,
- Pgno iFirst,
- Pgno iLast
+ LsmPgno iFirst,
+ LsmPgno iLast
){
int i;
if( fsRunEndsBetween(&pLevel->lhs, pIgnore, iFirst, iLast) ){
return 1;
@@ -1731,17 +1731,17 @@
Snapshot *pSnapshot, /* Worker snapshot */
Segment *pIgnore, /* Ignore this run when searching */
int iBlk /* Block number of block to free */
){
int rc = LSM_OK; /* Return code */
- Pgno iFirst; /* First page on block iBlk */
- Pgno iLast; /* Last page on block iBlk */
+ LsmPgno iFirst; /* First page on block iBlk */
+ LsmPgno iLast; /* Last page on block iBlk */
Level *pLevel; /* Used to iterate through levels */
int iIn; /* Used to iterate through append points */
int iOut = 0; /* Used to output append points */
- Pgno *aApp = pSnapshot->aiAppend;
+ LsmPgno *aApp = pSnapshot->aiAppend;
iFirst = fsFirstPageOnBlock(pFS, iBlk);
iLast = fsLastPageOnBlock(pFS, iBlk);
/* Check if any other run in the snapshot has a start or end page
@@ -1809,15 +1809,20 @@
/*
** aPgno is an array containing nPgno page numbers. Return the smallest page
** number from the array that falls on block iBlk. Or, if none of the pages
** in aPgno[] fall on block iBlk, return 0.
*/
-static Pgno firstOnBlock(FileSystem *pFS, int iBlk, Pgno *aPgno, int nPgno){
- Pgno iRet = 0;
+static LsmPgno firstOnBlock(
+ FileSystem *pFS,
+ int iBlk,
+ LsmPgno *aPgno,
+ int nPgno
+){
+ LsmPgno iRet = 0;
int i;
for(i=0; ipRedirect, iPg));
}
/*
** Return true if the second argument is not NULL and any of the first
@@ -1852,11 +1857,11 @@
** the new first page of the run).
*/
void lsmFsGobble(
lsm_db *pDb,
Segment *pRun,
- Pgno *aPgno,
+ LsmPgno *aPgno,
int nPgno
){
int rc = LSM_OK;
FileSystem *pFS = pDb->pFS;
Snapshot *pSnapshot = pDb->pWorker;
@@ -1869,11 +1874,11 @@
iBlk = fsPageToBlock(pFS, pRun->iFirst);
pRun->nSize += (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
while( rc==LSM_OK ){
int iNext = 0;
- Pgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
+ LsmPgno iFirst = firstOnBlock(pFS, iBlk, aPgno, nPgno);
if( iFirst ){
pRun->iFirst = iFirst;
break;
}
rc = fsBlockNext(pFS, pRun, iBlk, &iNext);
@@ -1903,15 +1908,15 @@
** But take block overflow and redirection into account.
*/
static int fsNextPageOffset(
FileSystem *pFS, /* File system object */
Segment *pSeg, /* Segment to move within */
- Pgno iPg, /* Offset of current page */
+ LsmPgno iPg, /* Offset of current page */
int nByte, /* Size of current page including headers */
- Pgno *piNext /* OUT: Offset of next page. Or zero (EOF) */
+ LsmPgno *piNext /* OUT: Offset of next page. Or zero (EOF) */
){
- Pgno iNext;
+ LsmPgno iNext;
int rc;
assert( pFS->pCompress );
rc = fsAddOffset(pFS, pSeg, iPg, nByte-1, &iNext);
@@ -1937,12 +1942,12 @@
** *piPrev is undefined.
*/
static int fsGetPageBefore(
FileSystem *pFS,
Segment *pSeg,
- Pgno iPg,
- Pgno *piPrev
+ LsmPgno iPg,
+ LsmPgno *piPrev
){
u8 aSz[3];
int rc;
i64 iRead;
@@ -1988,11 +1993,11 @@
** caller using lsmFsPageRelease().
*/
int lsmFsDbPageNext(Segment *pRun, Page *pPg, int eDir, Page **ppNext){
int rc = LSM_OK;
FileSystem *pFS = pPg->pFS;
- Pgno iPg = pPg->iPg;
+ LsmPgno iPg = pPg->iPg;
assert( 0==fsSegmentRedirects(pFS, pRun) );
if( pFS->pCompress ){
int nSpace = pPg->nCompress + 2*3;
@@ -2060,14 +2065,14 @@
**
** If argument pLvl is not NULL, then this function will not attempt to
** start the new segment immediately following any segment that is part
** of the right-hand-side of pLvl.
*/
-static Pgno findAppendPoint(FileSystem *pFS, Level *pLvl){
+static LsmPgno findAppendPoint(FileSystem *pFS, Level *pLvl){
int i;
- Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
- Pgno iRet = 0;
+ LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+ LsmPgno iRet = 0;
for(i=LSM_APPLIST_SZ-1; iRet==0 && i>=0; i--){
if( (iRet = aiAppend[i]) ){
if( pLvl ){
int iBlk = fsPageToBlock(pFS, iRet);
@@ -2096,14 +2101,14 @@
int bDefer,
Page **ppOut
){
int rc = LSM_OK;
Page *pPg = 0;
- Pgno iApp = 0;
- Pgno iNext = 0;
+ LsmPgno iApp = 0;
+ LsmPgno iNext = 0;
Segment *p = &pLvl->lhs;
- Pgno iPrev = p->iLastPg;
+ LsmPgno iPrev = p->iLastPg;
*ppOut = 0;
assert( p->pRedirect==0 );
if( pFS->pCompress || bDefer ){
@@ -2193,11 +2198,11 @@
** Otherwise, add the first free page in the last block used by the run
** to the lAppend list.
*/
if( fsLastPageOnPagesBlock(pFS, p->iLastPg)!=p->iLastPg ){
int i;
- Pgno *aiAppend = pFS->pDb->pWorker->aiAppend;
+ LsmPgno *aiAppend = pFS->pDb->pWorker->aiAppend;
for(i=0; iiLastPg+1;
break;
}
@@ -2224,11 +2229,11 @@
/*
** Obtain a reference to page number iPg.
**
** Return LSM_OK if successful, or an lsm error code if an error occurs.
*/
-int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, Pgno iPg, Page **ppPg){
+int lsmFsDbPageGet(FileSystem *pFS, Segment *pSeg, LsmPgno iPg, Page **ppPg){
return fsPageGet(pFS, pSeg, iPg, 0, ppPg, 0);
}
/*
** Obtain a reference to the last page in the segment passed as the
@@ -2236,11 +2241,11 @@
**
** Return LSM_OK if successful, or an lsm error code if an error occurs.
*/
int lsmFsDbPageLast(FileSystem *pFS, Segment *pSeg, Page **ppPg){
int rc;
- Pgno iPg = pSeg->iLastPg;
+ LsmPgno iPg = pSeg->iLastPg;
if( pFS->pCompress ){
int nSpace;
iPg++;
do {
nSpace = 0;
@@ -2364,18 +2369,18 @@
*/
static void fsMovePage(
FileSystem *pFS, /* File system object */
int iTo, /* Destination block */
int iFrom, /* Source block */
- Pgno *piPg /* IN/OUT: Page number */
+ LsmPgno *piPg /* IN/OUT: Page number */
){
- Pgno iPg = *piPg;
+ LsmPgno iPg = *piPg;
if( iFrom==fsPageToBlock(pFS, iPg) ){
const int nPagePerBlock = (
pFS->pCompress ? pFS ->nBlocksize : (pFS->nBlocksize / pFS->nPagesize)
);
- *piPg = iPg - (Pgno)(iFrom - iTo) * nPagePerBlock;
+ *piPg = iPg - (LsmPgno)(iFrom - iTo) * nPagePerBlock;
}
}
/*
** Copy the contents of block iFrom to block iTo.
@@ -2455,25 +2460,25 @@
** data is written to (this may be used as the page number if the data
** being appended is a new page record).
**
** This function is only used in compressed database mode.
*/
-static Pgno fsAppendData(
+static LsmPgno fsAppendData(
FileSystem *pFS, /* File-system handle */
Segment *pSeg, /* Segment to append to */
const u8 *aData, /* Buffer containing data to write */
int nData, /* Size of buffer aData[] in bytes */
int *pRc /* IN/OUT: Error code */
){
- Pgno iRet = 0;
+ LsmPgno iRet = 0;
int rc = *pRc;
assert( pFS->pCompress );
if( rc==LSM_OK ){
int nRem = 0;
int nWrite = 0;
- Pgno iLastOnBlock;
- Pgno iApp = pSeg->iLastPg+1;
+ LsmPgno iLastOnBlock;
+ LsmPgno iApp = pSeg->iLastPg+1;
/* If this is the first data written into the segment, find an append-point
** or allocate a new block. */
if( iApp==1 ){
pSeg->iFirst = iApp = findAppendPoint(pFS, 0);
@@ -2517,11 +2522,11 @@
rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iApp, aPtr, sizeof(aPtr));
}
/* Set the "prev" pointer on the new block */
if( rc==LSM_OK ){
- Pgno iWrite;
+ LsmPgno iWrite;
lsmPutU32(aPtr, fsPageToBlock(pFS, iApp));
iWrite = fsFirstPageOnBlock(pFS, iBlk);
rc = lsmEnvWrite(pFS->pEnv, pFS->fdDb, iWrite-4, aPtr, sizeof(aPtr));
if( nRem>0 ) iApp = iWrite;
}
@@ -2586,15 +2591,15 @@
** output variables is undefined.
*/
static int fsAppendPage(
FileSystem *pFS,
Segment *pSeg,
- Pgno *piNew,
+ LsmPgno *piNew,
int *piPrev,
int *piNext
){
- Pgno iPrev = pSeg->iLastPg;
+ LsmPgno iPrev = pSeg->iLastPg;
int rc;
assert( iPrev!=0 );
*piPrev = 0;
*piNext = 0;
@@ -2648,11 +2653,11 @@
}
/*
** If there exists a hash-table entry associated with page iPg, remove it.
*/
-static void fsRemoveHashEntry(FileSystem *pFS, Pgno iPg){
+static void fsRemoveHashEntry(FileSystem *pFS, LsmPgno iPg){
Page *p;
int iHash = fsHashKey(pFS->nHash, iPg);
for(p=pFS->apHash[iHash]; p && p->iPg!=iPg; p=p->pHashNext);
@@ -2801,13 +2806,13 @@
FileSystem *pFS,
Snapshot *pSnapshot,
Segment *pSeg
){
int rc = LSM_OK;
- if( pFS->pCompress ){
- Pgno iLast2;
- Pgno iLast = pSeg->iLastPg; /* Current last page of segment */
+ if( pFS->pCompress && pSeg->iFirst ){
+ LsmPgno iLast2;
+ LsmPgno iLast = pSeg->iLastPg; /* Current last page of segment */
int nPad; /* Bytes of padding required */
u8 aSz[3];
iLast2 = (1 + iLast/pFS->szSector) * pFS->szSector - 1;
assert( fsPageToBlock(pFS, iLast)==fsPageToBlock(pFS, iLast2) );
@@ -2933,19 +2938,19 @@
}
/*
** Helper function for lsmInfoArrayStructure().
*/
-static Segment *startsWith(Segment *pRun, Pgno iFirst){
+static Segment *startsWith(Segment *pRun, LsmPgno iFirst){
return (iFirst==pRun->iFirst) ? pRun : 0;
}
/*
** Return the segment that starts with page iFirst, if any. If no such segment
** can be found, return NULL.
*/
-static Segment *findSegment(Snapshot *pWorker, Pgno iFirst){
+static Segment *findSegment(Snapshot *pWorker, LsmPgno iFirst){
Level *pLvl; /* Used to iterate through db levels */
Segment *pSeg = 0; /* Pointer to segment to return */
for(pLvl=lsmDbSnapshotLevel(pWorker); pLvl && pSeg==0; pLvl=pLvl->pNext){
if( 0==(pSeg = startsWith(&pLvl->lhs, iFirst)) ){
@@ -2968,11 +2973,11 @@
** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
*/
int lsmInfoArrayStructure(
lsm_db *pDb,
int bBlock, /* True for block numbers only */
- Pgno iFirst,
+ LsmPgno iFirst,
char **pzOut
){
int rc = LSM_OK;
Snapshot *pWorker; /* Worker snapshot */
Segment *pArray = 0; /* Array to report on */
@@ -3033,11 +3038,11 @@
}
int lsmFsSegmentContainsPg(
FileSystem *pFS,
Segment *pSeg,
- Pgno iPg,
+ LsmPgno iPg,
int *pbRes
){
Redirect *pRedir = pSeg->pRedirect;
int rc = LSM_OK;
int iBlk;
@@ -3062,11 +3067,11 @@
** containing the array structure and LSM_OK is returned. The caller should
** eventually free the string using lsmFree().
**
** If an error occurs, *pzOut is set to NULL and an LSM error code returned.
*/
-int lsmInfoArrayPages(lsm_db *pDb, Pgno iFirst, char **pzOut){
+int lsmInfoArrayPages(lsm_db *pDb, LsmPgno iFirst, char **pzOut){
int rc = LSM_OK;
Snapshot *pWorker; /* Worker snapshot */
Segment *pSeg = 0; /* Array to report on */
int bUnlock = 0;
@@ -3295,13 +3300,13 @@
** Return true if pPg happens to be the last page in segment pSeg. Or false
** otherwise. This function is only invoked as part of assert() conditions.
*/
int lsmFsDbPageIsLast(Segment *pSeg, Page *pPg){
if( pPg->pFS->pCompress ){
- Pgno iNext = 0;
+ LsmPgno iNext = 0;
int rc;
rc = fsNextPageOffset(pPg->pFS, pSeg, pPg->iPg, pPg->nCompress+6, &iNext);
return (rc!=LSM_OK || iNext==0);
}
return (pPg->iPg==pSeg->iLastPg);
}
#endif
Index: ext/lsm1/lsm_main.c
==================================================================
--- ext/lsm1/lsm_main.c
+++ ext/lsm1/lsm_main.c
@@ -581,26 +581,26 @@
rc = lsmStructList(pDb, pzVal);
break;
}
case LSM_INFO_ARRAY_STRUCTURE: {
- Pgno pgno = va_arg(ap, Pgno);
+ LsmPgno pgno = va_arg(ap, LsmPgno);
char **pzVal = va_arg(ap, char **);
rc = lsmInfoArrayStructure(pDb, 0, pgno, pzVal);
break;
}
case LSM_INFO_ARRAY_PAGES: {
- Pgno pgno = va_arg(ap, Pgno);
+ LsmPgno pgno = va_arg(ap, LsmPgno);
char **pzVal = va_arg(ap, char **);
rc = lsmInfoArrayPages(pDb, pgno, pzVal);
break;
}
case LSM_INFO_PAGE_HEX_DUMP:
case LSM_INFO_PAGE_ASCII_DUMP: {
- Pgno pgno = va_arg(ap, Pgno);
+ LsmPgno pgno = va_arg(ap, LsmPgno);
char **pzVal = va_arg(ap, char **);
int bUnlock = 0;
rc = infoGetWorker(pDb, 0, &bUnlock);
if( rc==LSM_OK ){
int bHex = (eParam==LSM_INFO_PAGE_HEX_DUMP);
@@ -681,11 +681,11 @@
int nBefore;
int nAfter;
int nDiff;
if( nQuant>pDb->nTreeLimit ){
- nQuant = pDb->nTreeLimit;
+ nQuant = LSM_MAX(pDb->nTreeLimit, pgsz);
}
nBefore = lsmTreeSize(pDb);
if( bDeleteRange ){
rc = lsmTreeDelete(pDb, (void *)pKey, nKey, (void *)pVal, nVal);
Index: ext/lsm1/lsm_shared.c
==================================================================
--- ext/lsm1/lsm_shared.c
+++ ext/lsm1/lsm_shared.c
@@ -338,13 +338,10 @@
int rc;
/* Obtain a pointer to the shared-memory header */
assert( pDb->pShmhdr==0 );
assert( pDb->bReadonly==0 );
- rc = lsmShmCacheChunks(pDb, 1);
- if( rc!=LSM_OK ) return rc;
- pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
/* Block for an exclusive lock on DMS1. This lock serializes all calls
** to doDbConnect() and doDbDisconnect() across all processes. */
while( 1 ){
rc = lsmShmLock(pDb, LSM_LOCK_DMS1, LSM_LOCK_EXCL, 1);
@@ -351,14 +348,15 @@
if( rc!=LSM_BUSY ) break;
lsmEnvSleep(pDb->pEnv, nUs);
nUs = nUs * 2;
if( nUs>nUsMax ) nUs = nUsMax;
}
- if( rc!=LSM_OK ){
- pDb->pShmhdr = 0;
- return rc;
+ if( rc==LSM_OK ){
+ rc = lsmShmCacheChunks(pDb, 1);
}
+ if( rc!=LSM_OK ) return rc;
+ pDb->pShmhdr = (ShmHeader *)pDb->apShm[0];
/* Try an exclusive lock on DMS2/DMS3. If successful, this is the first
** and only connection to the database. In this case initialize the
** shared-memory and run log file recovery. */
assert( LSM_LOCK_DMS3==1+LSM_LOCK_DMS2 );
@@ -520,17 +518,15 @@
/* If the db handle is read-write, then connect to the system now. Run
** recovery as necessary. Or, if this is a read-only database handle,
** defer attempting to connect to the system until a read-transaction
** is opened. */
- if( pDb->bReadonly==0 ){
- if( rc==LSM_OK ){
- rc = lsmFsConfigure(pDb);
- }
- if( rc==LSM_OK ){
- rc = doDbConnect(pDb);
- }
+ if( rc==LSM_OK ){
+ rc = lsmFsConfigure(pDb);
+ }
+ if( rc==LSM_OK && pDb->bReadonly==0 ){
+ rc = doDbConnect(pDb);
}
return rc;
}
Index: ext/lsm1/lsm_sorted.c
==================================================================
--- ext/lsm1/lsm_sorted.c
+++ ext/lsm1/lsm_sorted.c
@@ -90,11 +90,11 @@
#define SEGMENT_NRECORD_OFFSET(pgsz) ((pgsz) - 2)
#define SEGMENT_FLAGS_OFFSET(pgsz) ((pgsz) - 2 - 2)
#define SEGMENT_POINTER_OFFSET(pgsz) ((pgsz) - 2 - 2 - 8)
#define SEGMENT_CELLPTR_OFFSET(pgsz, iCell) ((pgsz) - 2 - 2 - 8 - 2 - (iCell)*2)
-#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry)
+#define SEGMENT_EOF(pgsz, nEntry) SEGMENT_CELLPTR_OFFSET(pgsz, nEntry-1)
#define SEGMENT_BTREE_FLAG 0x0001
#define PGFTR_SKIP_NEXT_FLAG 0x0002
#define PGFTR_SKIP_THIS_FLAG 0x0004
@@ -102,13 +102,13 @@
#ifndef LSM_SEGMENTPTR_FREE_THRESHOLD
# define LSM_SEGMENTPTR_FREE_THRESHOLD 1024
#endif
typedef struct SegmentPtr SegmentPtr;
-typedef struct Blob Blob;
+typedef struct LsmBlob LsmBlob;
-struct Blob {
+struct LsmBlob {
lsm_env *pEnv;
void *pData;
int nData;
int nAlloc;
};
@@ -127,22 +127,22 @@
/* Current page. See segmentPtrLoadPage(). */
Page *pPg; /* Current page */
u16 flags; /* Copy of page flags field */
int nCell; /* Number of cells on pPg */
- Pgno iPtr; /* Base cascade pointer */
+ LsmPgno iPtr; /* Base cascade pointer */
/* Current cell. See segmentPtrLoadCell() */
int iCell; /* Current record within page pPg */
int eType; /* Type of current record */
- Pgno iPgPtr; /* Cascade pointer offset */
+ LsmPgno iPgPtr; /* Cascade pointer offset */
void *pKey; int nKey; /* Key associated with current record */
void *pVal; int nVal; /* Current record value (eType==WRITE only) */
/* Blobs used to allocate buffers for pKey and pVal as required */
- Blob blob1;
- Blob blob2;
+ LsmBlob blob1;
+ LsmBlob blob2;
};
/*
** Used to iterate through the keys stored in a b-tree hierarchy from start
** to finish. Only First() and Next() operations are required.
@@ -169,14 +169,14 @@
/* Cache of current entry. pKey==0 for EOF. */
void *pKey;
int nKey;
int eType;
- Pgno iPtr;
+ LsmPgno iPtr;
/* Storage for key, if not local */
- Blob blob;
+ LsmBlob blob;
};
/*
** A cursor used for merged searches or iterations through up to one
@@ -201,12 +201,12 @@
lsm_db *pDb; /* Connection that owns this cursor */
MultiCursor *pNext; /* Next cursor owned by connection pDb */
int flags; /* Mask of CURSOR_XXX flags */
int eType; /* Cache of current key type */
- Blob key; /* Cache of current key (or NULL) */
- Blob val; /* Cache of current value */
+ LsmBlob key; /* Cache of current key (or NULL) */
+ LsmBlob val; /* Cache of current value */
/* All the component cursors: */
TreeCursor *apTreeCsr[2]; /* Up to two tree cursors */
int iFree; /* Next element of free-list (-ve for eof) */
SegmentPtr *aPtr; /* Array of segment pointers */
@@ -219,11 +219,11 @@
/* Used by cursors flushing the in-memory tree only */
void *pSystemVal; /* Pointer to buffer to free */
/* Used by worker cursors only */
- Pgno *pPrevMergePtr;
+ LsmPgno *pPrevMergePtr;
};
/*
** The following constants are used to assign integers to each component
** cursor of a multi-cursor.
@@ -293,15 +293,15 @@
MultiCursor *pCsr; /* Cursor to read new segment contents from */
int bFlush; /* True if this is an in-memory tree flush */
Hierarchy hier; /* B-tree hierarchy under construction */
Page *pPage; /* Current output page */
int nWork; /* Number of calls to mergeWorkerNextPage() */
- Pgno *aGobble; /* Gobble point for each input segment */
+ LsmPgno *aGobble; /* Gobble point for each input segment */
- Pgno iIndirect;
+ LsmPgno iIndirect;
struct SavedPgno {
- Pgno iPgno;
+ LsmPgno iPgno;
int bStore;
} aSave[2];
};
#ifdef LSM_DEBUG_EXPENSIVE
@@ -369,11 +369,11 @@
aOut[5] = (u8)((nVal>>16) & 0xFF);
aOut[6] = (u8)((nVal>> 8) & 0xFF);
aOut[7] = (u8)((nVal ) & 0xFF);
}
-static int sortedBlobGrow(lsm_env *pEnv, Blob *pBlob, int nData){
+static int sortedBlobGrow(lsm_env *pEnv, LsmBlob *pBlob, int nData){
assert( pBlob->pEnv==pEnv || (pBlob->pEnv==0 && pBlob->pData==0) );
if( pBlob->nAllocpData = lsmReallocOrFree(pEnv, pBlob->pData, nData);
if( !pBlob->pData ) return LSM_NOMEM_BKPT;
pBlob->nAlloc = nData;
@@ -380,36 +380,36 @@
pBlob->pEnv = pEnv;
}
return LSM_OK;
}
-static int sortedBlobSet(lsm_env *pEnv, Blob *pBlob, void *pData, int nData){
+static int sortedBlobSet(lsm_env *pEnv, LsmBlob *pBlob, void *pData, int nData){
if( sortedBlobGrow(pEnv, pBlob, nData) ) return LSM_NOMEM;
memcpy(pBlob->pData, pData, nData);
pBlob->nData = nData;
return LSM_OK;
}
#if 0
-static int sortedBlobCopy(Blob *pDest, Blob *pSrc){
+static int sortedBlobCopy(LsmBlob *pDest, LsmBlob *pSrc){
return sortedBlobSet(pDest, pSrc->pData, pSrc->nData);
}
#endif
-static void sortedBlobFree(Blob *pBlob){
+static void sortedBlobFree(LsmBlob *pBlob){
assert( pBlob->pEnv || pBlob->pData==0 );
if( pBlob->pData ) lsmFree(pBlob->pEnv, pBlob->pData);
- memset(pBlob, 0, sizeof(Blob));
+ memset(pBlob, 0, sizeof(LsmBlob));
}
static int sortedReadData(
Segment *pSeg,
Page *pPg,
int iOff,
int nByte,
void **ppData,
- Blob *pBlob
+ LsmBlob *pBlob
){
int rc = LSM_OK;
int iEnd;
int nData;
int nCell;
@@ -479,12 +479,12 @@
static int pageGetNRec(u8 *aData, int nData){
return (int)lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
}
-static Pgno pageGetPtr(u8 *aData, int nData){
- return (Pgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
+static LsmPgno pageGetPtr(u8 *aData, int nData){
+ return (LsmPgno)lsmGetU64(&aData[SEGMENT_POINTER_OFFSET(nData)]);
}
static int pageGetFlags(u8 *aData, int nData){
return (int)lsmGetU16(&aData[SEGMENT_FLAGS_OFFSET(nData)]);
}
@@ -504,12 +504,12 @@
/*
** Return the decoded (possibly relative) pointer value stored in cell
** iCell from page aData/nData.
*/
-static Pgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
- Pgno iRet; /* Return value */
+static LsmPgno pageGetRecordPtr(u8 *aData, int nData, int iCell){
+ LsmPgno iRet; /* Return value */
u8 *aCell; /* Pointer to cell iCell */
assert( iCell=0 );
aCell = pageGetCell(aData, nData, iCell);
lsmVarintGet64(&aCell[1], &iRet);
@@ -520,11 +520,11 @@
Segment *pSeg, /* Segment pPg belongs to */
Page *pPg, /* Page to read from */
int iCell, /* Index of cell on page to read */
int *piTopic, /* OUT: Topic associated with this key */
int *pnKey, /* OUT: Size of key in bytes */
- Blob *pBlob /* If required, use this for dynamic memory */
+ LsmBlob *pBlob /* If required, use this for dynamic memory */
){
u8 *pKey;
int nDummy;
int eType;
u8 *aData;
@@ -552,11 +552,11 @@
lsm_env *pEnv, /* Environment handle */
Segment *pSeg, /* Segment pPg belongs to */
Page *pPg, /* Page to read from */
int iCell, /* Index of cell on page to read */
int *piTopic, /* OUT: Topic associated with this key */
- Blob *pBlob /* If required, use this for dynamic memory */
+ LsmBlob *pBlob /* If required, use this for dynamic memory */
){
int rc = LSM_OK;
int nKey;
u8 *aKey;
@@ -567,12 +567,12 @@
}
return rc;
}
-static Pgno pageGetBtreeRef(Page *pPg, int iKey){
- Pgno iRef;
+static LsmPgno pageGetBtreeRef(Page *pPg, int iKey){
+ LsmPgno iRef;
u8 *aData;
int nData;
u8 *aCell;
aData = fsPageData(pPg, &nData);
@@ -590,15 +590,15 @@
static int pageGetBtreeKey(
Segment *pSeg, /* Segment page pPg belongs to */
Page *pPg,
int iKey,
- Pgno *piPtr,
+ LsmPgno *piPtr,
int *piTopic,
void **ppKey,
int *pnKey,
- Blob *pBlob
+ LsmBlob *pBlob
){
u8 *aData;
int nData;
u8 *aCell;
int eType;
@@ -611,11 +611,11 @@
eType = *aCell++;
aCell += GETVARINT64(aCell, *piPtr);
if( eType==0 ){
int rc;
- Pgno iRef; /* Page number of referenced page */
+ LsmPgno iRef; /* Page number of referenced page */
Page *pRef;
aCell += GETVARINT64(aCell, iRef);
rc = lsmFsDbPageGet(lsmPageFS(pPg), pSeg, iRef, &pRef);
if( rc!=LSM_OK ) return rc;
pageGetKeyCopy(lsmPageEnv(pPg), pSeg, pRef, 0, &eType, pBlob);
@@ -636,11 +636,11 @@
if( pCsr->iPg<0 ){
pCsr->pKey = 0;
pCsr->nKey = 0;
pCsr->eType = 0;
}else{
- Pgno dummy;
+ LsmPgno dummy;
int iPg = pCsr->iPg;
int iCell = pCsr->aPg[iPg].iCell;
while( iCell<0 && (--iPg)>=0 ){
iCell = pCsr->aPg[iPg].iCell-1;
}
@@ -681,11 +681,11 @@
aData = fsPageData(pPg->pPage, &nData);
nCell = pageGetNRec(aData, nData);
assert( pPg->iCell<=nCell );
pPg->iCell++;
if( pPg->iCell==nCell ){
- Pgno iLoad;
+ LsmPgno iLoad;
/* Up to parent. */
lsmFsPageRelease(pPg->pPage);
pPg->pPage = 0;
pCsr->iPg--;
@@ -840,11 +840,11 @@
int rc = LSM_OK;
if( p->iPg ){
lsm_env *pEnv = lsmFsEnv(pCsr->pFS);
int iCell; /* Current cell number on leaf page */
- Pgno iLeaf; /* Page number of current leaf page */
+ LsmPgno iLeaf; /* Page number of current leaf page */
int nDepth; /* Depth of b-tree structure */
Segment *pSeg = pCsr->pSeg;
/* Decode the MergeInput structure */
iLeaf = p->iPg;
@@ -864,11 +864,11 @@
rc = lsmFsDbPageGet(pCsr->pFS, pSeg, iLeaf, pp);
}
/* Populate any other aPg[] array entries */
if( rc==LSM_OK && nDepth>1 ){
- Blob blob = {0,0,0};
+ LsmBlob blob = {0,0,0};
void *pSeek;
int nSeek;
int iTopicSeek;
int iPg = 0;
int iLoad = (int)pSeg->iRoot;
@@ -881,11 +881,11 @@
assert( iCell==-1 );
iTopicSeek = 1000;
pSeek = 0;
nSeek = 0;
}else{
- Pgno dummy;
+ LsmPgno dummy;
rc = pageGetBtreeKey(pSeg, pPg,
0, &dummy, &iTopicSeek, &pSeek, &nSeek, &pCsr->blob
);
}
@@ -910,11 +910,11 @@
while( iMax>=iMin ){
int iTry = (iMin+iMax)/2;
void *pKey; int nKey; /* Key for cell iTry */
int iTopic; /* Topic for key pKeyT/nKeyT */
- Pgno iPtr; /* Pointer for cell iTry */
+ LsmPgno iPtr; /* Pointer for cell iTry */
int res; /* (pSeek - pKeyT) */
rc = pageGetBtreeKey(
pSeg, pPg2, iTry, &iPtr, &iTopic, &pKey, &nKey, &blob
);
@@ -953,11 +953,11 @@
pBtreePg = &pCsr->aPg[pCsr->iPg];
aData = fsPageData(pBtreePg->pPage, &nData);
pCsr->iPtr = btreeCursorPtr(aData, nData, pBtreePg->iCell+1);
if( pBtreePg->iCell<0 ){
- Pgno dummy;
+ LsmPgno dummy;
int i;
for(i=pCsr->iPg-1; i>=0; i--){
if( pCsr->aPg[i].iCell>0 ) break;
}
assert( i>=0 );
@@ -1028,11 +1028,11 @@
static int segmentPtrReadData(
SegmentPtr *pPtr,
int iOff,
int nByte,
void **ppData,
- Blob *pBlob
+ LsmBlob *pBlob
){
return sortedReadData(pPtr->pSeg, pPtr->pPg, iOff, nByte, ppData, pBlob);
}
static int segmentPtrNextPage(
@@ -1121,19 +1121,19 @@
if( rc==LSM_OK ){
rc = lsmFsDbPageGet(pDb->pFS, pSeg, pMerge->splitkey.iPg, &pPg);
}
if( rc==LSM_OK ){
int iTopic;
- Blob blob = {0, 0, 0, 0};
+ LsmBlob blob = {0, 0, 0, 0};
u8 *aData;
int nData;
aData = lsmFsPageData(pPg, &nData);
if( pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG ){
void *pKey;
int nKey;
- Pgno dummy;
+ LsmPgno dummy;
rc = pageGetBtreeKey(pSeg,
pPg, pMerge->splitkey.iCell, &dummy, &iTopic, &pKey, &nKey, &blob
);
if( rc==LSM_OK && blob.pData!=pKey ){
rc = sortedBlobSet(pEnv, &blob, pKey, nKey);
@@ -1340,11 +1340,11 @@
MultiCursor *pCsr,
SegmentPtr *pPtr,
void *pKey, int nKey
){
lsm_env *pEnv = lsmFsEnv(pCsr->pDb->pFS);
- Blob blob = {0, 0, 0};
+ LsmBlob blob = {0, 0, 0};
int eDir;
int iTopic = 0; /* TODO: Fix me */
for(eDir=-1; eDir<=1; eDir+=2){
Page *pTest = pPtr->pPg;
@@ -1486,11 +1486,11 @@
static int ptrFwdPointer(
Page *pPage,
int iCell,
Segment *pSeg,
- Pgno *piPtr,
+ LsmPgno *piPtr,
int *pbFound
){
Page *pPg = pPage;
int iFirst = iCell;
int rc = LSM_OK;
@@ -1571,18 +1571,18 @@
** the big range-delete.
*/
static int segmentPtrFwdPointer(
MultiCursor *pCsr, /* Multi-cursor pPtr belongs to */
SegmentPtr *pPtr, /* Segment-pointer to extract FC ptr from */
- Pgno *piPtr /* OUT: FC pointer value */
+ LsmPgno *piPtr /* OUT: FC pointer value */
){
Level *pLvl = pPtr->pLevel;
Level *pNext = pLvl->pNext;
Page *pPg = pPtr->pPg;
int rc;
int bFound;
- Pgno iOut = 0;
+ LsmPgno iOut = 0;
if( pPtr->pSeg==&pLvl->lhs || pPtr->pSeg==&pLvl->aRhs[pLvl->nRight-1] ){
if( pNext==0
|| (pNext->nRight==0 && pNext->lhs.iRoot)
|| (pNext->nRight!=0 && pNext->aRhs[0].iRoot)
@@ -1639,11 +1639,11 @@
int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp;
int res = 0; /* Result of comparison operation */
int rc = LSM_OK;
int iMin;
int iMax;
- Pgno iPtrOut = 0;
+ LsmPgno iPtrOut = 0;
/* If the current page contains an oversized entry, then there are no
** pointers to one or more of the subsequent pages in the sorted run.
** The following call ensures that the segment-ptr points to the correct
** page in this case. */
@@ -1766,22 +1766,22 @@
static int seekInBtree(
MultiCursor *pCsr, /* Multi-cursor object */
Segment *pSeg, /* Seek within this segment */
int iTopic,
void *pKey, int nKey, /* Key to seek to */
- Pgno *aPg, /* OUT: Page numbers */
+ LsmPgno *aPg, /* OUT: Page numbers */
Page **ppPg /* OUT: Leaf (sorted-run) page reference */
){
int i = 0;
int rc;
int iPg;
Page *pPg = 0;
- Blob blob = {0, 0, 0};
+ LsmBlob blob = {0, 0, 0};
iPg = (int)pSeg->iRoot;
do {
- Pgno *piFirst = 0;
+ LsmPgno *piFirst = 0;
if( aPg ){
aPg[i++] = iPg;
piFirst = &aPg[i];
}
@@ -1806,11 +1806,11 @@
iMax = nRec-1;
while( iMax>=iMin ){
int iTry = (iMin+iMax)/2;
void *pKeyT; int nKeyT; /* Key for cell iTry */
int iTopicT; /* Topic for key pKeyT/nKeyT */
- Pgno iPtr; /* Pointer associated with cell iTry */
+ LsmPgno iPtr; /* Pointer associated with cell iTry */
int res; /* (pKey - pKeyT) */
rc = pageGetBtreeKey(
pSeg, pPg, iTry, &iPtr, &iTopicT, &pKeyT, &nKeyT, &blob
);
@@ -1897,11 +1897,11 @@
MultiCursor *pCsr, /* Sorted cursor object to seek */
SegmentPtr *aPtr, /* Pointer to array of (nRhs+1) SPs */
int eSeek, /* Search bias - see above */
int iTopic, /* Key topic to search for */
void *pKey, int nKey, /* Key to search for */
- Pgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */
+ LsmPgno *piPgno, /* IN/OUT: fraction cascade pointer (or 0) */
int *pbStop /* OUT: See above */
){
Level *pLvl = aPtr[0].pLevel; /* Level to seek within */
int rc = LSM_OK; /* Return code */
int iOut = 0; /* Pointer to return to caller */
@@ -1920,25 +1920,30 @@
/* If (res<0), then key pKey/nKey is smaller than the split-key (or this
** is not a composite level and there is no split-key). Search the
** left-hand-side of the level in this case. */
if( res<0 ){
+ int i;
int iPtr = 0;
if( nRhs==0 ) iPtr = (int)*piPgno;
rc = seekInSegment(
pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop
);
if( rc==LSM_OK && nRhs>0 && eSeek==LSM_SEEK_GE && aPtr[0].pPg==0 ){
res = 0;
}
+ for(i=1; i<=nRhs; i++){
+ segmentPtrReset(&aPtr[i], LSM_SEGMENTPTR_FREE_THRESHOLD);
+ }
}
if( res>=0 ){
int bHit = 0; /* True if at least one rhs is not EOF */
int iPtr = (int)*piPgno;
int i;
+ segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD);
for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){
SegmentPtr *pPtr = &aPtr[i];
iOut = 0;
rc = seekInSegment(
pCsr, pPtr, iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop
@@ -2866,11 +2871,11 @@
static int multiCursorEnd(MultiCursor *pCsr, int bLast){
int rc = LSM_OK;
int i;
- pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK);
+ pCsr->flags &= ~(CURSOR_NEXT_OK | CURSOR_PREV_OK | CURSOR_SEEK_EQ);
pCsr->flags |= (bLast ? CURSOR_PREV_OK : CURSOR_NEXT_OK);
pCsr->iFree = 0;
/* Position the two in-memory tree cursors */
for(i=0; rc==LSM_OK && i<2; i++){
@@ -3053,11 +3058,11 @@
){
int eESeek = eSeek; /* Effective eSeek parameter */
int bStop = 0; /* Set to true to halt search operation */
int rc = LSM_OK; /* Return code */
int iPtr = 0; /* Used to iterate through pCsr->aPtr[] */
- Pgno iPgno = 0; /* FC pointer value */
+ LsmPgno iPgno = 0; /* FC pointer value */
assert( pCsr->apTreeCsr[0]==0 || iTopic==0 );
assert( pCsr->apTreeCsr[1]==0 || iTopic==0 );
if( eESeek==LSM_SEEK_LEFAST ) eESeek = LSM_SEEK_LE;
@@ -3535,11 +3540,11 @@
** 1. The record format is (usually, see below) as follows:
**
** + Type byte (always SORTED_SEPARATOR or SORTED_SYSTEM_SEPARATOR),
** + Absolute pointer value (varint),
** + Number of bytes in key (varint),
-** + Blob containing key data.
+** + LsmBlob containing key data.
**
** 2. All pointer values are stored as absolute values (not offsets
** relative to the footer pointer value).
**
** 3. Each pointer that is part of a record points to a page that
@@ -3569,12 +3574,12 @@
*/
static int mergeWorkerBtreeWrite(
MergeWorker *pMW,
u8 eType,
- Pgno iPtr,
- Pgno iKeyPg,
+ LsmPgno iPtr,
+ LsmPgno iKeyPg,
void *pKey,
int nKey
){
Hierarchy *p = &pMW->hier;
lsm_db *pDb = pMW->pDb; /* Database handle */
@@ -3680,11 +3685,11 @@
}
static int mergeWorkerBtreeIndirect(MergeWorker *pMW){
int rc = LSM_OK;
if( pMW->iIndirect ){
- Pgno iKeyPg = pMW->aSave[1].iPgno;
+ LsmPgno iKeyPg = pMW->aSave[1].iPgno;
rc = mergeWorkerBtreeWrite(pMW, 0, pMW->iIndirect, iKeyPg, 0, 0);
pMW->iIndirect = 0;
}
return rc;
}
@@ -3701,11 +3706,11 @@
int iTopic, /* Topic value for this key */
void *pKey, /* Pointer to key buffer */
int nKey /* Size of pKey buffer in bytes */
){
int rc = LSM_OK; /* Return Code */
- Pgno iPtr; /* Pointer value to accompany pKey/nKey */
+ LsmPgno iPtr; /* Pointer value to accompany pKey/nKey */
assert( pMW->aSave[0].bStore==0 );
assert( pMW->aSave[1].bStore==0 );
rc = mergeWorkerBtreeIndirect(pMW);
@@ -3732,11 +3737,11 @@
static int mergeWorkerFinishHierarchy(
MergeWorker *pMW /* Merge worker object */
){
int i; /* Used to loop through apHier[] */
int rc = LSM_OK; /* Return code */
- Pgno iPtr; /* New right-hand-child pointer value */
+ LsmPgno iPtr; /* New right-hand-child pointer value */
iPtr = pMW->aSave[0].iPgno;
for(i=0; ihier.nHier && rc==LSM_OK; i++){
Page *pPg = pMW->hier.apHier[i];
int nData; /* Size of aData[] in bytes */
@@ -3828,11 +3833,11 @@
**
** If successful, LSM_OK is returned. Otherwise, an error code.
*/
static int mergeWorkerNextPage(
MergeWorker *pMW, /* Merge worker object to append page to */
- Pgno iFPtr /* Pointer value for footer of new page */
+ LsmPgno iFPtr /* Pointer value for footer of new page */
){
int rc = LSM_OK; /* Return code */
Page *pNext = 0; /* New page appended to run */
lsm_db *pDb = pMW->pDb; /* Database handle */
@@ -3997,10 +4002,15 @@
/* If the entire header will not fit on page pPg, or if page pPg is
** marked read-only, advance to the next page of the output run. */
iOff = pMerge->iOutputOff;
if( iOff<0 || pPg==0 || iOff+nHdr > SEGMENT_EOF(nData, nRec+1) ){
+ if( iOff>=0 && pPg ){
+ /* Zero any free space on the page */
+ assert( aData );
+ memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
+ }
iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
iRPtr = iPtr - iFPtr;
iOff = 0;
nRec = 0;
rc = mergeWorkerNextPage(pMW, iFPtr);
@@ -4067,40 +4077,53 @@
MultiCursor *pCsr = pMW->pCsr;
/* Unless the merge has finished, save the cursor position in the
** Merge.aInput[] array. See function mergeWorkerInit() for the
** code to restore a cursor position based on aInput[]. */
- if( rc==LSM_OK && pCsr && lsmMCursorValid(pCsr) ){
+ if( rc==LSM_OK && pCsr ){
Merge *pMerge = pMW->pLevel->pMerge;
- int bBtree = (pCsr->pBtCsr!=0);
- int iPtr;
-
- /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
- assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
- assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
-
- for(i=0; i<(pMerge->nInput-bBtree); i++){
- SegmentPtr *pPtr = &pCsr->aPtr[i];
- if( pPtr->pPg ){
- pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
- pMerge->aInput[i].iCell = pPtr->iCell;
- }else{
- pMerge->aInput[i].iPg = 0;
- pMerge->aInput[i].iCell = 0;
- }
- }
- if( bBtree && pMerge->nInput ){
- assert( i==pCsr->nPtr );
- btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
- }
-
- /* Store the location of the split-key */
- iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
- if( iPtrnPtr ){
- pMerge->splitkey = pMerge->aInput[iPtr];
- }else{
- btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
+ if( lsmMCursorValid(pCsr) ){
+ int bBtree = (pCsr->pBtCsr!=0);
+ int iPtr;
+
+ /* pMerge->nInput==0 indicates that this is a FlushTree() operation. */
+ assert( pMerge->nInput==0 || pMW->pLevel->nRight>0 );
+ assert( pMerge->nInput==0 || pMerge->nInput==(pCsr->nPtr+bBtree) );
+
+ for(i=0; i<(pMerge->nInput-bBtree); i++){
+ SegmentPtr *pPtr = &pCsr->aPtr[i];
+ if( pPtr->pPg ){
+ pMerge->aInput[i].iPg = lsmFsPageNumber(pPtr->pPg);
+ pMerge->aInput[i].iCell = pPtr->iCell;
+ }else{
+ pMerge->aInput[i].iPg = 0;
+ pMerge->aInput[i].iCell = 0;
+ }
+ }
+ if( bBtree && pMerge->nInput ){
+ assert( i==pCsr->nPtr );
+ btreeCursorPosition(pCsr->pBtCsr, &pMerge->aInput[i]);
+ }
+
+ /* Store the location of the split-key */
+ iPtr = pCsr->aTree[1] - CURSOR_DATA_SEGMENT;
+ if( iPtrnPtr ){
+ pMerge->splitkey = pMerge->aInput[iPtr];
+ }else{
+ btreeCursorSplitkey(pCsr->pBtCsr, &pMerge->splitkey);
+ }
+ }
+
+ /* Zero any free space left on the final page. This helps with
+ ** compression if using a compression hook. And prevents valgrind
+ ** from complaining about uninitialized byte passed to write(). */
+ if( pMW->pPage ){
+ int nData;
+ u8 *aData = fsPageData(pMW->pPage, &nData);
+ int iOff = pMerge->iOutputOff;
+ int iEof = SEGMENT_EOF(nData, pageGetNRec(aData, nData));
+ memset(&aData[iOff], 0, iEof - iOff);
}
pMerge->iOutputOff = -1;
}
@@ -4198,11 +4221,11 @@
lsm_db *pDb = pMW->pDb; /* Database handle */
MultiCursor *pCsr; /* Cursor to read input data from */
int rc = LSM_OK; /* Return code */
int eType; /* SORTED_SEPARATOR, WRITE or DELETE */
void *pKey; int nKey; /* Key */
- Pgno iPtr;
+ LsmPgno iPtr;
int iVal;
pCsr = pMW->pCsr;
/* Pull the next record out of the source cursor. */
@@ -4351,11 +4374,11 @@
}
if( rc!=LSM_OK ){
lsmMCursorClose(pCsr, 0);
}else{
- Pgno iLeftPtr = 0;
+ LsmPgno iLeftPtr = 0;
Merge merge; /* Merge object used to create new level */
MergeWorker mergeworker; /* MergeWorker object for the same purpose */
memset(&merge, 0, sizeof(Merge));
memset(&mergeworker, 0, sizeof(MergeWorker));
@@ -4528,11 +4551,11 @@
assert( pLevel->nRight>0 );
memset(pMW, 0, sizeof(MergeWorker));
pMW->pDb = pDb;
pMW->pLevel = pLevel;
- pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno) * pLevel->nRight, &rc);
+ pMW->aGobble = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*pLevel->nRight,&rc);
/* Create a multi-cursor to read the data to write to the new
** segment. The new segment contains:
**
** 1. Records from LHS of each of the nMerge levels being merged.
@@ -4610,20 +4633,20 @@
int iGobble /* pCsr->aPtr[] entry to operate on */
){
int rc = LSM_OK;
if( rtTopic(pCsr->eType)==0 ){
Segment *pSeg = pCsr->aPtr[iGobble].pSeg;
- Pgno *aPg;
+ LsmPgno *aPg;
int nPg;
/* Seek from the root of the b-tree to the segment leaf that may contain
** a key equal to the one multi-cursor currently points to. Record the
** page number of each b-tree page and the leaf. The segment may be
** gobbled up to (but not including) the first of these page numbers.
*/
assert( pSeg->iRoot>0 );
- aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(Pgno)*32, &rc);
+ aPg = lsmMallocZeroRc(pDb->pEnv, sizeof(LsmPgno)*32, &rc);
if( rc==LSM_OK ){
rc = seekInBtree(pCsr, pSeg,
rtTopic(pCsr->eType), pCsr->key.pData, pCsr->key.nData, aPg, 0
);
}
@@ -5232,20 +5255,19 @@
}
/* If the in-memory part of the free-list is too large, write a new
** top-level containing just the in-memory free-list entries to disk. */
if( rc==LSM_OK && pDb->pWorker->freelist.nEntry > pDb->nMaxFreelist ){
- int nPg = 0;
while( rc==LSM_OK && lsmDatabaseFull(pDb) ){
+ int nPg = 0;
rc = sortedWork(pDb, 16, nMerge, 1, &nPg);
nRem -= nPg;
}
if( rc==LSM_OK ){
rc = sortedNewFreelistOnly(pDb);
}
- nRem -= nPg;
- if( nPg ) bDirty = 1;
+ bDirty = 1;
}
if( rc==LSM_OK ){
*pnWrite = (nMax - nRem);
*pbCkpt = (bCkpt && nRem<=0);
@@ -5446,13 +5468,13 @@
** Space for the returned string is allocated using lsmMalloc(), and should
** be freed by the caller using lsmFree().
*/
static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
int nSize = pSeg->nSize;
- Pgno iRoot = pSeg->iRoot;
- Pgno iFirst = pSeg->iFirst;
- Pgno iLast = pSeg->iLastPg;
+ LsmPgno iRoot = pSeg->iRoot;
+ LsmPgno iFirst = pSeg->iFirst;
+ LsmPgno iLast = pSeg->iLastPg;
char *z;
char *z1;
char *z2;
int nPad;
@@ -5507,11 +5529,11 @@
return i;
}
void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
- Blob blob = {0, 0, 0}; /* Blob used for keys */
+ LsmBlob blob = {0, 0, 0}; /* LsmBlob used for keys */
LsmString s;
int i;
int nRec;
int iPtr;
@@ -5543,11 +5565,11 @@
eType = *aCell++;
assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 );
aCell += lsmVarintGet32(aCell, &iPgPtr);
if( eType==0 ){
- Pgno iRef; /* Page number of referenced page */
+ LsmPgno iRef; /* Page number of referenced page */
aCell += lsmVarintGet64(aCell, &iRef);
lsmFsDbPageGet(pDb->pFS, pRun, iRef, &pRef);
aKey = pageGetKey(pRun, pRef, 0, &iTopic, &nKey, &blob);
}else{
aCell += lsmVarintGet32(aCell, &nKey);
@@ -5587,11 +5609,11 @@
int iCell,
int *peType,
int *piPgPtr,
u8 **paKey, int *pnKey,
u8 **paVal, int *pnVal,
- Blob *pBlob
+ LsmBlob *pBlob
){
u8 *aData; int nData; /* Page data */
u8 *aKey; int nKey = 0; /* Key */
u8 *aVal = 0; int nVal = 0; /* Value */
int eType;
@@ -5605,11 +5627,11 @@
eType = *aCell++;
aCell += lsmVarintGet32(aCell, &iPgPtr);
if( eType==0 ){
int dummy;
- Pgno iRef; /* Page number of referenced page */
+ LsmPgno iRef; /* Page number of referenced page */
aCell += lsmVarintGet64(aCell, &iRef);
if( bIndirect ){
lsmFsDbPageGet(pDb->pFS, pSeg, iRef, &pRef);
pageGetKeyCopy(pDb->pEnv, pSeg, pRef, 0, &dummy, pBlob);
aKey = (u8 *)pBlob->pData;
@@ -5651,11 +5673,11 @@
#define INFO_PAGE_DUMP_HEX 0x04
#define INFO_PAGE_DUMP_INDIRECT 0x08
static int infoPageDump(
lsm_db *pDb, /* Database handle */
- Pgno iPg, /* Page number of page to dump */
+ LsmPgno iPg, /* Page number of page to dump */
int flags,
char **pzOut /* OUT: lsmMalloc'd string */
){
int rc = LSM_OK; /* Return code */
Page *pPg = 0; /* Handle for page iPg */
@@ -5692,11 +5714,11 @@
if( rc==LSM_OK ){
rc = lsmFsDbPageGet(pDb->pFS, 0, iPg, &pPg);
}
if( rc==LSM_OK ){
- Blob blob = {0, 0, 0, 0};
+ LsmBlob blob = {0, 0, 0, 0};
int nKeyWidth = 0;
LsmString str;
int nRec;
int iPtr;
int flags2;
@@ -5727,11 +5749,11 @@
for(iCell=0; iCellpFS, pSeg, pSeg->iFirst, &pPg);
while( pPg ){
u8 *aData; int nData;
Page *pNext;
@@ -6032,11 +6054,11 @@
int bRhs /* True if pTwo may have been Gobble()d */
){
int rc = LSM_OK; /* Error code */
SegmentPtr ptr1; /* Iterates through pOne */
SegmentPtr ptr2; /* Iterates through pTwo */
- Pgno iPrev;
+ LsmPgno iPrev;
assert( pOne && pTwo );
memset(&ptr1, 0, sizeof(ptr1));
memset(&ptr2, 0, sizeof(ptr1));
@@ -6055,11 +6077,11 @@
if( rc==LSM_OK && ptr1.nCell>0 ){
rc = segmentPtrLoadCell(&ptr1, 0);
}
while( rc==LSM_OK && ptr2.pPg ){
- Pgno iThis;
+ LsmPgno iThis;
/* Advance to the next page of segment pTwo that contains at least
** one cell. Break out of the loop if the iterator reaches EOF. */
do{
rc = segmentPtrNextPage(&ptr2, 1);
@@ -6117,11 +6139,11 @@
lsm_db *pDb,
Segment *pSeg
){
int rc = LSM_OK; /* Return code */
if( pSeg->iRoot ){
- Blob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */
+ LsmBlob blob = {0, 0, 0}; /* Buffer used to cache overflow keys */
FileSystem *pFS = pDb->pFS; /* File system to read from */
Page *pPg = 0; /* Main run page */
BtreeCursor *pCsr = 0; /* Btree cursor */
rc = btreeCursorNew(pDb, pSeg, &pCsr);
ADDED ext/lsm1/tool/mklsm1c.tcl
Index: ext/lsm1/tool/mklsm1c.tcl
==================================================================
--- /dev/null
+++ ext/lsm1/tool/mklsm1c.tcl
@@ -0,0 +1,88 @@
+#!/bin/sh
+# restart with tclsh \
+exec tclsh "$0" "$@"
+
+set srcdir [file dirname [file dirname [info script]]]
+set G(src) [string map [list %dir% $srcdir] {
+ %dir%/lsm.h
+ %dir%/lsmInt.h
+ %dir%/lsm_vtab.c
+ %dir%/lsm_ckpt.c
+ %dir%/lsm_file.c
+ %dir%/lsm_log.c
+ %dir%/lsm_main.c
+ %dir%/lsm_mem.c
+ %dir%/lsm_mutex.c
+ %dir%/lsm_shared.c
+ %dir%/lsm_sorted.c
+ %dir%/lsm_str.c
+ %dir%/lsm_tree.c
+ %dir%/lsm_unix.c
+ %dir%/lsm_varint.c
+ %dir%/lsm_win32.c
+}]
+
+set G(hdr) {
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1)
+
+#if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+#endif
+#if defined(NDEBUG) && defined(SQLITE_DEBUG)
+# undef NDEBUG
+#endif
+
+}
+
+set G(footer) {
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_LSM1) */
+}
+
+#-------------------------------------------------------------------------
+# Read and return the entire contents of text file $zFile from disk.
+#
+proc readfile {zFile} {
+ set fd [open $zFile]
+ set data [read $fd]
+ close $fd
+ return $data
+}
+
+proc lsm1c_init {zOut} {
+ global G
+ set G(fd) stdout
+ set G(fd) [open $zOut w]
+
+ puts -nonewline $G(fd) $G(hdr)
+}
+
+proc lsm1c_printfile {zIn} {
+ global G
+ set data [readfile $zIn]
+ set zTail [file tail $zIn]
+ puts $G(fd) "#line 1 \"$zTail\""
+
+ foreach line [split $data "\n"] {
+ if {[regexp {^# *include.*lsm} $line]} {
+ set line "/* $line */"
+ } elseif { [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?lsm[^_]} $line] } {
+ set line "static $line"
+ }
+ puts $G(fd) $line
+ }
+}
+
+proc lsm1c_close {} {
+ global G
+ puts -nonewline $G(fd) $G(footer)
+ if {$G(fd)!="stdout"} {
+ close $G(fd)
+ }
+}
+
+
+lsm1c_init lsm1.c
+foreach f $G(src) { lsm1c_printfile $f }
+lsm1c_close
Index: ext/misc/README.md
==================================================================
--- ext/misc/README.md
+++ ext/misc/README.md
@@ -12,14 +12,23 @@
* **carray.c** — This module implements the
[carray](https://www.sqlite.org/carray.html) table-valued function.
It is a good example of how to go about implementing a custom
[table-valued function](https://www.sqlite.org/vtab.html#tabfunc2).
+ * **csv.c** — A [virtual table](https://sqlite.org/vtab.html)
+ for reading
+ [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values).
+
* **dbdump.c** — This is not actually a loadable extension, but
rather a library that implements an approximate equivalent to the
".dump" command of the
[command-line shell](https://www.sqlite.org/cli.html).
+
+ * **json1.c** — Various SQL functions and table-valued functions
+ for processing JSON. This extension is already built into the
+ [SQLite amalgamation](https://sqlite.org/amalgamation.html). See
+ for additional information.
* **memvfs.c** — This file implements a custom
[VFS](https://www.sqlite.org/vfs.html) that stores an entire database
file in a single block of RAM. It serves as a good example of how
to implement a simple custom VFS.
@@ -36,5 +45,16 @@
sha3_query() SQL functions. The file is named "shathree.c" instead
of "sha3.c" because the default entry point names in SQLite are based
on the source filename with digits removed, so if we used the name
"sha3.c" then the entry point would conflict with the prior "sha1.c"
extension.
+
+ * **unionvtab.c** — Implementation of the unionvtab and
+ [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables.
+ These virtual tables allow a single
+ large table to be spread out across multiple database files. In the
+ case of swarmvtab, the individual database files can be attached on
+ demand.
+
+ * **zipfile.c** — A [virtual table](https://sqlite.org/vtab.html)
+ that can read and write a
+ [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29).
Index: ext/misc/amatch.c
==================================================================
--- ext/misc/amatch.c
+++ ext/misc/amatch.c
@@ -1471,11 +1471,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
#endif /* SQLITE_OMIT_VIRTUALTABLE */
/*
ADDED ext/misc/appendvfs.c
Index: ext/misc/appendvfs.c
==================================================================
--- /dev/null
+++ ext/misc/appendvfs.c
@@ -0,0 +1,565 @@
+/*
+** 2017-10-20
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a VFS shim that allows an SQLite database to be
+** appended onto the end of some other file, such as an executable.
+**
+** A special record must appear at the end of the file that identifies the
+** file as an appended database and provides an offset to page 1. For
+** best performance page 1 should be located at a disk page boundary, though
+** that is not required.
+**
+** When opening a database using this VFS, the connection might treat
+** the file as an ordinary SQLite database, or it might treat is as a
+** database appended onto some other file. Here are the rules:
+**
+** (1) When opening a new empty file, that file is treated as an ordinary
+** database.
+**
+** (2) When opening a file that begins with the standard SQLite prefix
+** string "SQLite format 3", that file is treated as an ordinary
+** database.
+**
+** (3) When opening a file that ends with the appendvfs trailer string
+** "Start-Of-SQLite3-NNNNNNNN" that file is treated as an appended
+** database.
+**
+** (4) If none of the above apply and the SQLITE_OPEN_CREATE flag is
+** set, then a new database is appended to the already existing file.
+**
+** (5) Otherwise, SQLITE_CANTOPEN is returned.
+**
+** To avoid unnecessary complications with the PENDING_BYTE, the size of
+** the file containing the database is limited to 1GB. This VFS will refuse
+** to read or write past the 1GB mark. This restriction might be lifted in
+** future versions. For now, if you need a large database, then keep the
+** database in a separate file.
+**
+** If the file being opened is not an appended database, then this shim is
+** a pass-through into the default underlying VFS.
+**/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include
+#include
+
+/* The append mark at the end of the database is:
+**
+** Start-Of-SQLite3-NNNNNNNN
+** 123456789 123456789 12345
+**
+** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is
+** the offset to page 1.
+*/
+#define APND_MARK_PREFIX "Start-Of-SQLite3-"
+#define APND_MARK_PREFIX_SZ 17
+#define APND_MARK_SIZE 25
+
+/*
+** Maximum size of the combined prefix + database + append-mark. This
+** must be less than 0x40000000 to avoid locking issues on Windows.
+*/
+#define APND_MAX_SIZE (65536*15259)
+
+/*
+** Forward declaration of objects used by this utility
+*/
+typedef struct sqlite3_vfs ApndVfs;
+typedef struct ApndFile ApndFile;
+
+/* Access to a lower-level VFS that (might) implement dynamic loading,
+** access to randomness, etc.
+*/
+#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData))
+#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1))
+
+/* An open file */
+struct ApndFile {
+ sqlite3_file base; /* IO methods */
+ sqlite3_int64 iPgOne; /* File offset to page 1 */
+ sqlite3_int64 iMark; /* Start of the append-mark */
+};
+
+/*
+** Methods for ApndFile
+*/
+static int apndClose(sqlite3_file*);
+static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst);
+static int apndTruncate(sqlite3_file*, sqlite3_int64 size);
+static int apndSync(sqlite3_file*, int flags);
+static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int apndLock(sqlite3_file*, int);
+static int apndUnlock(sqlite3_file*, int);
+static int apndCheckReservedLock(sqlite3_file*, int *pResOut);
+static int apndFileControl(sqlite3_file*, int op, void *pArg);
+static int apndSectorSize(sqlite3_file*);
+static int apndDeviceCharacteristics(sqlite3_file*);
+static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**);
+static int apndShmLock(sqlite3_file*, int offset, int n, int flags);
+static void apndShmBarrier(sqlite3_file*);
+static int apndShmUnmap(sqlite3_file*, int deleteFlag);
+static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp);
+static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p);
+
+/*
+** Methods for ApndVfs
+*/
+static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *);
+static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir);
+static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *);
+static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut);
+static void *apndDlOpen(sqlite3_vfs*, const char *zFilename);
+static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg);
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void);
+static void apndDlClose(sqlite3_vfs*, void*);
+static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut);
+static int apndSleep(sqlite3_vfs*, int microseconds);
+static int apndCurrentTime(sqlite3_vfs*, double*);
+static int apndGetLastError(sqlite3_vfs*, int, char *);
+static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*);
+static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr);
+static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z);
+static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName);
+
+static sqlite3_vfs apnd_vfs = {
+ 3, /* iVersion (set when registered) */
+ 0, /* szOsFile (set when registered) */
+ 1024, /* mxPathname */
+ 0, /* pNext */
+ "apndvfs", /* zName */
+ 0, /* pAppData (set when registered) */
+ apndOpen, /* xOpen */
+ apndDelete, /* xDelete */
+ apndAccess, /* xAccess */
+ apndFullPathname, /* xFullPathname */
+ apndDlOpen, /* xDlOpen */
+ apndDlError, /* xDlError */
+ apndDlSym, /* xDlSym */
+ apndDlClose, /* xDlClose */
+ apndRandomness, /* xRandomness */
+ apndSleep, /* xSleep */
+ apndCurrentTime, /* xCurrentTime */
+ apndGetLastError, /* xGetLastError */
+ apndCurrentTimeInt64, /* xCurrentTimeInt64 */
+ apndSetSystemCall, /* xSetSystemCall */
+ apndGetSystemCall, /* xGetSystemCall */
+ apndNextSystemCall /* xNextSystemCall */
+};
+
+static const sqlite3_io_methods apnd_io_methods = {
+ 3, /* iVersion */
+ apndClose, /* xClose */
+ apndRead, /* xRead */
+ apndWrite, /* xWrite */
+ apndTruncate, /* xTruncate */
+ apndSync, /* xSync */
+ apndFileSize, /* xFileSize */
+ apndLock, /* xLock */
+ apndUnlock, /* xUnlock */
+ apndCheckReservedLock, /* xCheckReservedLock */
+ apndFileControl, /* xFileControl */
+ apndSectorSize, /* xSectorSize */
+ apndDeviceCharacteristics, /* xDeviceCharacteristics */
+ apndShmMap, /* xShmMap */
+ apndShmLock, /* xShmLock */
+ apndShmBarrier, /* xShmBarrier */
+ apndShmUnmap, /* xShmUnmap */
+ apndFetch, /* xFetch */
+ apndUnfetch /* xUnfetch */
+};
+
+
+
+/*
+** Close an apnd-file.
+*/
+static int apndClose(sqlite3_file *pFile){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xClose(pFile);
+}
+
+/*
+** Read data from an apnd-file.
+*/
+static int apndRead(
+ sqlite3_file *pFile,
+ void *zBuf,
+ int iAmt,
+ sqlite_int64 iOfst
+){
+ ApndFile *p = (ApndFile *)pFile;
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+}
+
+/*
+** Add the append-mark onto the end of the file.
+*/
+static int apndWriteMark(ApndFile *p, sqlite3_file *pFile){
+ int i;
+ unsigned char a[APND_MARK_SIZE];
+ memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ);
+ for(i=0; i<8; i++){
+ a[APND_MARK_PREFIX_SZ+i] = (p->iPgOne >> (56 - i*8)) & 0xff;
+ }
+ return pFile->pMethods->xWrite(pFile, a, APND_MARK_SIZE, p->iMark);
+}
+
+/*
+** Write data to an apnd-file.
+*/
+static int apndWrite(
+ sqlite3_file *pFile,
+ const void *zBuf,
+ int iAmt,
+ sqlite_int64 iOfst
+){
+ int rc;
+ ApndFile *p = (ApndFile *)pFile;
+ pFile = ORIGFILE(pFile);
+ if( iOfst+iAmt>=APND_MAX_SIZE ) return SQLITE_FULL;
+ rc = pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst+p->iPgOne);
+ if( rc==SQLITE_OK && iOfst + iAmt + p->iPgOne > p->iMark ){
+ sqlite3_int64 sz = 0;
+ rc = pFile->pMethods->xFileSize(pFile, &sz);
+ if( rc==SQLITE_OK ){
+ p->iMark = sz - APND_MARK_SIZE;
+ if( iOfst + iAmt + p->iPgOne > p->iMark ){
+ p->iMark = p->iPgOne + iOfst + iAmt;
+ rc = apndWriteMark(p, pFile);
+ }
+ }
+ }
+ return rc;
+}
+
+/*
+** Truncate an apnd-file.
+*/
+static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){
+ int rc;
+ ApndFile *p = (ApndFile *)pFile;
+ pFile = ORIGFILE(pFile);
+ rc = pFile->pMethods->xTruncate(pFile, size+p->iPgOne+APND_MARK_SIZE);
+ if( rc==SQLITE_OK ){
+ p->iMark = p->iPgOne+size;
+ rc = apndWriteMark(p, pFile);
+ }
+ return rc;
+}
+
+/*
+** Sync an apnd-file.
+*/
+static int apndSync(sqlite3_file *pFile, int flags){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xSync(pFile, flags);
+}
+
+/*
+** Return the current file-size of an apnd-file.
+*/
+static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){
+ ApndFile *p = (ApndFile *)pFile;
+ int rc;
+ pFile = ORIGFILE(p);
+ rc = pFile->pMethods->xFileSize(pFile, pSize);
+ if( rc==SQLITE_OK && p->iPgOne ){
+ *pSize -= p->iPgOne + APND_MARK_SIZE;
+ }
+ return rc;
+}
+
+/*
+** Lock an apnd-file.
+*/
+static int apndLock(sqlite3_file *pFile, int eLock){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xLock(pFile, eLock);
+}
+
+/*
+** Unlock an apnd-file.
+*/
+static int apndUnlock(sqlite3_file *pFile, int eLock){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xUnlock(pFile, eLock);
+}
+
+/*
+** Check if another file-handle holds a RESERVED lock on an apnd-file.
+*/
+static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xCheckReservedLock(pFile, pResOut);
+}
+
+/*
+** File control method. For custom operations on an apnd-file.
+*/
+static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){
+ ApndFile *p = (ApndFile *)pFile;
+ int rc;
+ pFile = ORIGFILE(pFile);
+ rc = pFile->pMethods->xFileControl(pFile, op, pArg);
+ if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){
+ *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", p->iPgOne, *(char**)pArg);
+ }
+ return rc;
+}
+
+/*
+** Return the sector-size in bytes for an apnd-file.
+*/
+static int apndSectorSize(sqlite3_file *pFile){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xSectorSize(pFile);
+}
+
+/*
+** Return the device characteristic flags supported by an apnd-file.
+*/
+static int apndDeviceCharacteristics(sqlite3_file *pFile){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xDeviceCharacteristics(pFile);
+}
+
+/* Create a shared memory file mapping */
+static int apndShmMap(
+ sqlite3_file *pFile,
+ int iPg,
+ int pgsz,
+ int bExtend,
+ void volatile **pp
+){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp);
+}
+
+/* Perform locking on a shared-memory segment */
+static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xShmLock(pFile,offset,n,flags);
+}
+
+/* Memory barrier operation on shared memory */
+static void apndShmBarrier(sqlite3_file *pFile){
+ pFile = ORIGFILE(pFile);
+ pFile->pMethods->xShmBarrier(pFile);
+}
+
+/* Unmap a shared memory segment */
+static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xShmUnmap(pFile,deleteFlag);
+}
+
+/* Fetch a page of a memory-mapped file */
+static int apndFetch(
+ sqlite3_file *pFile,
+ sqlite3_int64 iOfst,
+ int iAmt,
+ void **pp
+){
+ ApndFile *p = (ApndFile *)pFile;
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp);
+}
+
+/* Release a memory-mapped page */
+static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){
+ ApndFile *p = (ApndFile *)pFile;
+ pFile = ORIGFILE(pFile);
+ return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage);
+}
+
+/*
+** Check to see if the file is an ordinary SQLite database file.
+*/
+static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){
+ int rc;
+ char zHdr[16];
+ static const char aSqliteHdr[] = "SQLite format 3";
+ if( sz<512 ) return 0;
+ rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0);
+ if( rc ) return 0;
+ return memcmp(zHdr, aSqliteHdr, sizeof(zHdr))==0;
+}
+
+/*
+** Try to read the append-mark off the end of a file. Return the
+** start of the appended database if the append-mark is present. If
+** there is no append-mark, return -1;
+*/
+static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){
+ int rc, i;
+ sqlite3_int64 iMark;
+ unsigned char a[APND_MARK_SIZE];
+
+ if( sz<=APND_MARK_SIZE ) return -1;
+ rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE);
+ if( rc ) return -1;
+ if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1;
+ iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ]&0x7f))<<56;
+ for(i=1; i<8; i++){
+ iMark += (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]<<(56-8*i);
+ }
+ return iMark;
+}
+
+/*
+** Open an apnd file handle.
+*/
+static int apndOpen(
+ sqlite3_vfs *pVfs,
+ const char *zName,
+ sqlite3_file *pFile,
+ int flags,
+ int *pOutFlags
+){
+ ApndFile *p;
+ sqlite3_file *pSubFile;
+ sqlite3_vfs *pSubVfs;
+ int rc;
+ sqlite3_int64 sz;
+ pSubVfs = ORIGVFS(pVfs);
+ if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){
+ return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags);
+ }
+ p = (ApndFile*)pFile;
+ memset(p, 0, sizeof(*p));
+ pSubFile = ORIGFILE(pFile);
+ p->base.pMethods = &apnd_io_methods;
+ rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags);
+ if( rc ) goto apnd_open_done;
+ rc = pSubFile->pMethods->xFileSize(pSubFile, &sz);
+ if( rc ){
+ pSubFile->pMethods->xClose(pSubFile);
+ goto apnd_open_done;
+ }
+ if( apndIsOrdinaryDatabaseFile(sz, pSubFile) ){
+ memmove(pFile, pSubFile, pSubVfs->szOsFile);
+ return SQLITE_OK;
+ }
+ p->iMark = 0;
+ p->iPgOne = apndReadMark(sz, pFile);
+ if( p->iPgOne>0 ){
+ return SQLITE_OK;
+ }
+ if( (flags & SQLITE_OPEN_CREATE)==0 ){
+ pSubFile->pMethods->xClose(pSubFile);
+ rc = SQLITE_CANTOPEN;
+ }
+ p->iPgOne = (sz+0xfff) & ~(sqlite3_int64)0xfff;
+apnd_open_done:
+ if( rc ) pFile->pMethods = 0;
+ return rc;
+}
+
+/*
+** All other VFS methods are pass-thrus.
+*/
+static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
+ return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync);
+}
+static int apndAccess(
+ sqlite3_vfs *pVfs,
+ const char *zPath,
+ int flags,
+ int *pResOut
+){
+ return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut);
+}
+static int apndFullPathname(
+ sqlite3_vfs *pVfs,
+ const char *zPath,
+ int nOut,
+ char *zOut
+){
+ return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut);
+}
+static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){
+ return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath);
+}
+static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){
+ ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg);
+}
+static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){
+ return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym);
+}
+static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){
+ ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle);
+}
+static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){
+ return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut);
+}
+static int apndSleep(sqlite3_vfs *pVfs, int nMicro){
+ return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro);
+}
+static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
+ return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut);
+}
+static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){
+ return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b);
+}
+static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){
+ return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
+}
+static int apndSetSystemCall(
+ sqlite3_vfs *pVfs,
+ const char *zName,
+ sqlite3_syscall_ptr pCall
+){
+ return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall);
+}
+static sqlite3_syscall_ptr apndGetSystemCall(
+ sqlite3_vfs *pVfs,
+ const char *zName
+){
+ return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName);
+}
+static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){
+ return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName);
+}
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+/*
+** This routine is called when the extension is loaded.
+** Register the new VFS.
+*/
+int sqlite3_appendvfs_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ int rc = SQLITE_OK;
+ sqlite3_vfs *pOrig;
+ SQLITE_EXTENSION_INIT2(pApi);
+ (void)pzErrMsg;
+ (void)db;
+ pOrig = sqlite3_vfs_find(0);
+ apnd_vfs.iVersion = pOrig->iVersion;
+ apnd_vfs.pAppData = pOrig;
+ apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile);
+ rc = sqlite3_vfs_register(&apnd_vfs, 0);
+#ifdef APPENDVFS_TEST
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister);
+ }
+#endif
+ if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
+ return rc;
+}
Index: ext/misc/bgckpt.c
==================================================================
--- ext/misc/bgckpt.c
+++ ext/misc/bgckpt.c
@@ -229,10 +229,16 @@
return TCL_OK;
}
#endif /* SQLITE_TEST */
#else
-int Bgckpt_Init(Tcl_Interp *interp){
- return TCL_OK;
-}
+#if defined(INCLUDE_SQLITE_TCL_H)
+# include "sqlite_tcl.h"
+#else
+# include "tcl.h"
+# ifndef SQLITE_TCLAPI
+# define SQLITE_TCLAPI
+# endif
+#endif
+int Bgckpt_Init(Tcl_Interp *interp){ return TCL_OK; }
#endif
ADDED ext/misc/btreeinfo.c
Index: ext/misc/btreeinfo.c
==================================================================
--- /dev/null
+++ ext/misc/btreeinfo.c
@@ -0,0 +1,429 @@
+/*
+** 2017-10-24
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of the "sqlite_btreeinfo" virtual table.
+**
+** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual
+** table that shows information about all btrees in an SQLite database file.
+** The schema is like this:
+**
+** CREATE TABLE sqlite_btreeinfo(
+** type TEXT, -- "table" or "index"
+** name TEXT, -- Name of table or index for this btree.
+** tbl_name TEXT, -- Associated table
+** rootpage INT, -- The root page of the btree
+** sql TEXT, -- SQL for this btree - from sqlite_master
+** hasRowid BOOLEAN, -- True if the btree has a rowid
+** nEntry INT, -- Estimated number of enteries
+** nPage INT, -- Estimated number of pages
+** depth INT, -- Depth of the btree
+** szPage INT, -- Size of each page in bytes
+** zSchema TEXT HIDDEN -- The schema to which this btree belongs
+** );
+**
+** The first 5 fields are taken directly from the sqlite_master table.
+** Considering only the first 5 fields, the only difference between
+** this virtual table and the sqlite_master table is that this virtual
+** table omits all entries that have a 0 or NULL rowid - in other words
+** it omits triggers and views.
+**
+** The value added by this table comes in the next 5 fields.
+**
+** Note that nEntry and nPage are *estimated*. They are computed doing
+** a single search from the root to a leaf, counting the number of cells
+** at each level, and assuming that unvisited pages have a similar number
+** of cells.
+**
+** The sqlite_dbpage virtual table must be available for this virtual table
+** to operate.
+**
+** USAGE EXAMPLES:
+**
+** Show the table btrees in a schema order with the tables with the most
+** rows occuring first:
+**
+** SELECT name, nEntry
+** FROM sqlite_btreeinfo
+** WHERE type='table'
+** ORDER BY nEntry DESC, name;
+**
+** Show the names of all WITHOUT ROWID tables:
+**
+** SELECT name FROM sqlite_btreeinfo
+** WHERE type='table' AND NOT hasRowid;
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include
+#include
+
+/* Columns available in this virtual table */
+#define BINFO_COLUMN_TYPE 0
+#define BINFO_COLUMN_NAME 1
+#define BINFO_COLUMN_TBL_NAME 2
+#define BINFO_COLUMN_ROOTPAGE 3
+#define BINFO_COLUMN_SQL 4
+#define BINFO_COLUMN_HASROWID 5
+#define BINFO_COLUMN_NENTRY 6
+#define BINFO_COLUMN_NPAGE 7
+#define BINFO_COLUMN_DEPTH 8
+#define BINFO_COLUMN_SZPAGE 9
+#define BINFO_COLUMN_SCHEMA 10
+
+/* Forward declarations */
+typedef struct BinfoTable BinfoTable;
+typedef struct BinfoCursor BinfoCursor;
+
+/* A cursor for the sqlite_btreeinfo table */
+struct BinfoCursor {
+ sqlite3_vtab_cursor base; /* Base class. Must be first */
+ sqlite3_stmt *pStmt; /* Query against sqlite_master */
+ int rc; /* Result of previous sqlite_step() call */
+ int hasRowid; /* hasRowid value. Negative if unknown. */
+ sqlite3_int64 nEntry; /* nEntry value */
+ int nPage; /* nPage value */
+ int depth; /* depth value */
+ int szPage; /* size of a btree page. 0 if unknown */
+ char *zSchema; /* Schema being interrogated */
+};
+
+/* The sqlite_btreeinfo table */
+struct BinfoTable {
+ sqlite3_vtab base; /* Base class. Must be first */
+ sqlite3 *db; /* The databse connection */
+};
+
+/*
+** Connect to the sqlite_btreeinfo virtual table.
+*/
+static int binfoConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ BinfoTable *pTab = 0;
+ int rc = SQLITE_OK;
+ rc = sqlite3_declare_vtab(db,
+ "CREATE TABLE x(\n"
+ " type TEXT,\n"
+ " name TEXT,\n"
+ " tbl_name TEXT,\n"
+ " rootpage INT,\n"
+ " sql TEXT,\n"
+ " hasRowid BOOLEAN,\n"
+ " nEntry INT,\n"
+ " nPage INT,\n"
+ " depth INT,\n"
+ " szPage INT,\n"
+ " zSchema TEXT HIDDEN\n"
+ ")");
+ if( rc==SQLITE_OK ){
+ pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable));
+ if( pTab==0 ) rc = SQLITE_NOMEM;
+ }
+ assert( rc==SQLITE_OK || pTab==0 );
+ if( pTab ){
+ pTab->db = db;
+ }
+ *ppVtab = (sqlite3_vtab*)pTab;
+ return rc;
+}
+
+/*
+** Disconnect from or destroy a btreeinfo virtual table.
+*/
+static int binfoDisconnect(sqlite3_vtab *pVtab){
+ sqlite3_free(pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** idxNum:
+**
+** 0 Use "main" for the schema
+** 1 Schema identified by parameter ?1
+*/
+static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+ int i;
+ pIdxInfo->estimatedCost = 10000.0; /* Cost estimate */
+ pIdxInfo->estimatedRows = 100;
+ for(i=0; inConstraint; i++){
+ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+ if( p->usable
+ && p->iColumn==BINFO_COLUMN_SCHEMA
+ && p->op==SQLITE_INDEX_CONSTRAINT_EQ
+ ){
+ pIdxInfo->estimatedCost = 1000.0;
+ pIdxInfo->idxNum = 1;
+ pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[i].omit = 1;
+ break;
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+ BinfoCursor *pCsr;
+
+ pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor));
+ if( pCsr==0 ){
+ return SQLITE_NOMEM;
+ }else{
+ memset(pCsr, 0, sizeof(BinfoCursor));
+ pCsr->base.pVtab = pVTab;
+ }
+
+ *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int binfoClose(sqlite3_vtab_cursor *pCursor){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->zSchema);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int binfoNext(sqlite3_vtab_cursor *pCursor){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ pCsr->rc = sqlite3_step(pCsr->pStmt);
+ pCsr->hasRowid = -1;
+ return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int binfoEof(sqlite3_vtab_cursor *pCursor){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ return pCsr->rc!=SQLITE_ROW;
+}
+
+/* Position a cursor back to the beginning.
+*/
+static int binfoFilter(
+ sqlite3_vtab_cursor *pCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ BinfoTable *pTab = (BinfoTable *)pCursor->pVtab;
+ char *zSql;
+ int rc;
+
+ sqlite3_free(pCsr->zSchema);
+ if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){
+ pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+ }else{
+ pCsr->zSchema = sqlite3_mprintf("main");
+ }
+ zSql = sqlite3_mprintf(
+ "SELECT 0, 'table','sqlite_master','sqlite_master',1,NULL "
+ "UNION ALL "
+ "SELECT rowid, type, name, tbl_name, rootpage, sql"
+ " FROM \"%w\".sqlite_master WHERE rootpage>=1",
+ pCsr->zSchema);
+ sqlite3_finalize(pCsr->pStmt);
+ pCsr->pStmt = 0;
+ pCsr->hasRowid = -1;
+ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+ sqlite3_free(zSql);
+ if( rc==SQLITE_OK ){
+ rc = binfoNext(pCursor);
+ }
+ return rc;
+}
+
+/* Decode big-endian integers */
+static unsigned int get_uint16(unsigned char *a){
+ return (a[0]<<8)|a[1];
+}
+static unsigned int get_uint32(unsigned char *a){
+ return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3];
+}
+
+/* Examine the b-tree rooted at pgno and estimate its size.
+** Return non-zero if anything goes wrong.
+*/
+static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){
+ sqlite3_int64 nEntry = 1;
+ int nPage = 1;
+ unsigned char *aData;
+ sqlite3_stmt *pStmt = 0;
+ int rc = SQLITE_OK;
+ int pgsz = 0;
+ int nCell;
+ int iCell;
+
+ rc = sqlite3_prepare_v2(db,
+ "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1,
+ &pStmt, 0);
+ if( rc ) return rc;
+ pCsr->depth = 1;
+ while(1){
+ sqlite3_bind_int(pStmt, 1, pgno);
+ rc = sqlite3_step(pStmt);
+ if( rc!=SQLITE_ROW ){
+ rc = SQLITE_ERROR;
+ break;
+ }
+ pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0);
+ aData = (unsigned char*)sqlite3_column_blob(pStmt, 0);
+ if( aData==0 ){
+ rc = SQLITE_NOMEM;
+ break;
+ }
+ if( pgno==1 ){
+ aData += 100;
+ pgsz -= 100;
+ }
+ pCsr->hasRowid = aData[0]!=2 && aData[0]!=10;
+ nCell = get_uint16(aData+3);
+ nEntry *= (nCell+1);
+ if( aData[0]==10 || aData[0]==13 ) break;
+ nPage *= (nCell+1);
+ if( nCell<=1 ){
+ pgno = get_uint32(aData+8);
+ }else{
+ iCell = get_uint16(aData+12+2*(nCell/2));
+ if( pgno==1 ) iCell -= 100;
+ if( iCell<=12 || iCell>=pgsz-4 ){
+ rc = SQLITE_CORRUPT;
+ break;
+ }
+ pgno = get_uint32(aData+iCell);
+ }
+ pCsr->depth++;
+ sqlite3_reset(pStmt);
+ }
+ sqlite3_finalize(pStmt);
+ pCsr->nPage = nPage;
+ pCsr->nEntry = nEntry;
+ if( rc==SQLITE_ROW ) rc = SQLITE_OK;
+ return rc;
+}
+
+/* Return a column for the sqlite_btreeinfo table */
+static int binfoColumn(
+ sqlite3_vtab_cursor *pCursor,
+ sqlite3_context *ctx,
+ int i
+){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){
+ int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1);
+ sqlite3 *db = sqlite3_context_db_handle(ctx);
+ int rc = binfoCompute(db, pgno, pCsr);
+ if( rc ){
+ pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+ return SQLITE_ERROR;
+ }
+ }
+ switch( i ){
+ case BINFO_COLUMN_NAME:
+ case BINFO_COLUMN_TYPE:
+ case BINFO_COLUMN_TBL_NAME:
+ case BINFO_COLUMN_ROOTPAGE:
+ case BINFO_COLUMN_SQL: {
+ sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1));
+ break;
+ }
+ case BINFO_COLUMN_HASROWID: {
+ sqlite3_result_int(ctx, pCsr->hasRowid);
+ break;
+ }
+ case BINFO_COLUMN_NENTRY: {
+ sqlite3_result_int64(ctx, pCsr->nEntry);
+ break;
+ }
+ case BINFO_COLUMN_NPAGE: {
+ sqlite3_result_int(ctx, pCsr->nPage);
+ break;
+ }
+ case BINFO_COLUMN_DEPTH: {
+ sqlite3_result_int(ctx, pCsr->depth);
+ break;
+ }
+ case BINFO_COLUMN_SCHEMA: {
+ sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC);
+ break;
+ }
+ }
+ return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ BinfoCursor *pCsr = (BinfoCursor *)pCursor;
+ *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+ return SQLITE_OK;
+}
+
+/*
+** Invoke this routine to register the "sqlite_btreeinfo" virtual table module
+*/
+int sqlite3BinfoRegister(sqlite3 *db){
+ static sqlite3_module binfo_module = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ binfoConnect, /* xConnect */
+ binfoBestIndex, /* xBestIndex */
+ binfoDisconnect, /* xDisconnect */
+ 0, /* xDestroy */
+ binfoOpen, /* xOpen - open a cursor */
+ binfoClose, /* xClose - close a cursor */
+ binfoFilter, /* xFilter - configure scan constraints */
+ binfoNext, /* xNext - advance a cursor */
+ binfoEof, /* xEof - check for end of scan */
+ binfoColumn, /* xColumn - read data */
+ binfoRowid, /* xRowid - read data */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindMethod */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
+ };
+ return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_btreeinfo_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi);
+ return sqlite3BinfoRegister(db);
+}
Index: ext/misc/closure.c
==================================================================
--- ext/misc/closure.c
+++ ext/misc/closure.c
@@ -824,21 +824,16 @@
sqlite3_index_info *pIdxInfo /* Information about the query */
){
int iPlan = 0;
int i;
int idx = 1;
- int seenMatch = 0;
const struct sqlite3_index_constraint *pConstraint;
closure_vtab *pVtab = (closure_vtab*)pTab;
double rCost = 10000000.0;
pConstraint = pIdxInfo->aConstraint;
for(i=0; inConstraint; i++, pConstraint++){
- if( pConstraint->iColumn==CLOSURE_COL_ROOT
- && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
- seenMatch = 1;
- }
if( pConstraint->usable==0 ) continue;
if( (iPlan & 1)==0
&& pConstraint->iColumn==CLOSURE_COL_ROOT
&& pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ
){
@@ -890,19 +885,30 @@
){
/* All of tablename, idcolumn, and parentcolumn must be specified
** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints
** or else the result is an empty set. */
iPlan = 0;
+ }
+ if( (iPlan&1)==0 ){
+ /* If there is no usable "root=?" term, then set the index-type to 0.
+ ** Also clear any argvIndex variables already set. This is necessary
+ ** to prevent the core from throwing an "xBestIndex malfunction error"
+ ** error (because the argvIndex values are not contiguously assigned
+ ** starting from 1). */
+ rCost *= 1e30;
+ for(i=0; inConstraint; i++, pConstraint++){
+ pIdxInfo->aConstraintUsage[i].argvIndex = 0;
+ }
+ iPlan = 0;
}
pIdxInfo->idxNum = iPlan;
if( pIdxInfo->nOrderBy==1
&& pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID
&& pIdxInfo->aOrderBy[0].desc==0
){
pIdxInfo->orderByConsumed = 1;
}
- if( seenMatch && (iPlan&1)==0 ) rCost *= 1e30;
pIdxInfo->estimatedCost = rCost;
return SQLITE_OK;
}
@@ -930,11 +936,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
#endif /* SQLITE_OMIT_VIRTUALTABLE */
/*
Index: ext/misc/completion.c
==================================================================
--- ext/misc/completion.c
+++ ext/misc/completion.c
@@ -60,10 +60,11 @@
sqlite3 *db; /* Database connection for this cursor */
int nPrefix, nLine; /* Number of bytes in zPrefix and zLine */
char *zPrefix; /* The prefix for the word we want to complete */
char *zLine; /* The whole that we want to complete */
const char *zCurrentRow; /* Current output row */
+ int szRow; /* Length of the zCurrentRow string */
sqlite3_stmt *pStmt; /* Current statement */
sqlite3_int64 iRowid; /* The rowid */
int ePhase; /* Current phase */
int j; /* inter-phase counter */
};
@@ -76,11 +77,11 @@
#define COMPLETION_FUNCTIONS 3
#define COMPLETION_COLLATIONS 4
#define COMPLETION_INDEXES 5
#define COMPLETION_TRIGGERS 6
#define COMPLETION_DATABASES 7
-#define COMPLETION_TABLES 8
+#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */
#define COMPLETION_COLUMNS 9
#define COMPLETION_MODULES 10
#define COMPLETION_EOF 11
/*
@@ -172,36 +173,10 @@
completionCursorReset((completion_cursor*)cur);
sqlite3_free(cur);
return SQLITE_OK;
}
-/*
-** All SQL keywords understood by SQLite
-*/
-static const char *completionKwrds[] = {
- "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
- "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
- "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
- "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
- "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
- "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
- "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
- "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
- "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
- "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
- "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
- "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
- "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
- "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
- "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
- "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
- "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
- "WITH", "WITHOUT",
-};
-#define completionKwCount \
- (int)(sizeof(completionKwrds)/sizeof(completionKwrds[0]))
-
/*
** Advance a completion_cursor to its next row of output.
**
** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object
** record the current state of the scan. This routine sets ->zCurrentRow
@@ -220,15 +195,15 @@
int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */
pCur->iRowid++;
while( pCur->ePhase!=COMPLETION_EOF ){
switch( pCur->ePhase ){
case COMPLETION_KEYWORDS: {
- if( pCur->j >= completionKwCount ){
+ if( pCur->j >= sqlite3_keyword_count() ){
pCur->zCurrentRow = 0;
pCur->ePhase = COMPLETION_DATABASES;
}else{
- pCur->zCurrentRow = completionKwrds[pCur->j++];
+ sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow);
}
iCol = -1;
break;
}
case COMPLETION_DATABASES: {
@@ -248,12 +223,11 @@
sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0);
while( sqlite3_step(pS2)==SQLITE_ROW ){
const char *zDb = (const char*)sqlite3_column_text(pS2, 1);
zSql = sqlite3_mprintf(
"%z%s"
- "SELECT name FROM \"%w\".sqlite_master"
- " WHERE type='table'",
+ "SELECT name FROM \"%w\".sqlite_master",
zSql, zSep, zDb
);
if( zSql==0 ) return SQLITE_NOMEM;
zSep = " UNION ";
}
@@ -297,20 +271,23 @@
if( pCur->zCurrentRow==0 ) continue;
}else{
if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){
/* Extract the next row of content */
pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol);
+ pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol);
}else{
/* When all rows are finished, advance to the next phase */
sqlite3_finalize(pCur->pStmt);
pCur->pStmt = 0;
pCur->ePhase = eNextPhase;
continue;
}
}
if( pCur->nPrefix==0 ) break;
- if( sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 ){
+ if( pCur->nPrefix<=pCur->szRow
+ && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0
+ ){
break;
}
}
return SQLITE_OK;
@@ -326,11 +303,11 @@
int i /* Which column to return */
){
completion_cursor *pCur = (completion_cursor*)cur;
switch( i ){
case COMPLETION_COLUMN_CANDIDATE: {
- sqlite3_result_text(ctx, pCur->zCurrentRow, -1, SQLITE_TRANSIENT);
+ sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT);
break;
}
case COMPLETION_COLUMN_PREFIX: {
sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT);
break;
@@ -386,19 +363,18 @@
pCur->nPrefix = sqlite3_value_bytes(argv[iArg]);
if( pCur->nPrefix>0 ){
pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
if( pCur->zPrefix==0 ) return SQLITE_NOMEM;
}
- iArg++;
+ iArg = 1;
}
if( idxNum & 2 ){
pCur->nLine = sqlite3_value_bytes(argv[iArg]);
if( pCur->nLine>0 ){
pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg]));
if( pCur->zLine==0 ) return SQLITE_NOMEM;
}
- iArg++;
}
if( pCur->zLine!=0 && pCur->zPrefix==0 ){
int i = pCur->nLine;
while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){
i--;
@@ -490,11 +466,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
#endif /* SQLITE_OMIT_VIRTUALTABLE */
int sqlite3CompletionVtabInit(sqlite3 *db){
Index: ext/misc/compress.c
==================================================================
--- ext/misc/compress.c
+++ ext/misc/compress.c
@@ -25,10 +25,25 @@
** is the input size in bytes (the size of X before compression). The
** variable-length integer is implemented as 1 to 5 bytes. There are
** seven bits per integer stored in the lower seven bits of each byte.
** More significant bits occur first. The most significant bit (0x80)
** is a flag to indicate the end of the integer.
+**
+** This function, SQLAR, and ZIP all use the same "deflate" compression
+** algorithm, but each is subtly different:
+**
+** * ZIP uses raw deflate.
+**
+** * SQLAR uses the "zlib format" which is raw deflate with a two-byte
+** algorithm-identification header and a four-byte checksum at the end.
+**
+** * This utility uses the "zlib format" like SQLAR, but adds the variable-
+** length integer uncompressed size value at the beginning.
+**
+** This function might be extended in the future to support compression
+** formats other than deflate, by providing a different algorithm-id
+** mark following the variable-length integer size parameter.
*/
static void compressFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
Index: ext/misc/csv.c
==================================================================
--- ext/misc/csv.c
+++ ext/misc/csv.c
@@ -17,13 +17,13 @@
**
** .load ./csv
** CREATE VIRTUAL TABLE temp.csv USING csv(filename=FILENAME);
** SELECT * FROM csv;
**
-** The columns are named "c1", "c2", "c3", ... by default. But the
-** application can define its own CREATE TABLE statement as an additional
-** parameter. For example:
+** The columns are named "c1", "c2", "c3", ... by default. Or the
+** application can define its own CREATE TABLE statement using the
+** schema= parameter, like this:
**
** CREATE VIRTUAL TABLE temp.csv2 USING csv(
** filename = "../http.log",
** schema = "CREATE TABLE x(date,ipaddr,url,referrer,userAgent)"
** );
@@ -30,13 +30,13 @@
**
** Instead of specifying a file, the text of the CSV can be loaded using
** the data= parameter.
**
** If the columns=N parameter is supplied, then the CSV file is assumed to have
-** N columns. If the columns parameter is omitted, the CSV file is opened
-** as soon as the virtual table is constructed and the first row of the CSV
-** is read in order to count the tables.
+** N columns. If both the columns= and schema= parameters are omitted, then
+** the number and names of the columns is determined by the first line of
+** the CSV input.
**
** Some extra debugging features (used for testing virtual tables) are available
** if this module is compiled with -DSQLITE_TEST.
*/
#include
@@ -130,10 +130,11 @@
csv_errmsg(p, "out of memory");
return 1;
}
p->in = fopen(zFilename, "rb");
if( p->in==0 ){
+ sqlite3_free(p->zIn);
csv_reader_reset(p);
csv_errmsg(p, "cannot open '%s' for reading", zFilename);
return 1;
}
}else{
@@ -202,19 +203,20 @@
** from sqlite3_malloc64().
** + Keep track of the line number in p->nLine.
** + Store the character that terminates the field in p->cTerm. Store
** EOF on end-of-file.
**
-** Return "" at EOF. Return 0 on an OOM error.
+** Return 0 at EOF or on OOM. On EOF, the p->cTerm character will have
+** been set to EOF.
*/
static char *csv_read_one_field(CsvReader *p){
int c;
p->n = 0;
c = csv_getc(p);
if( c==EOF ){
p->cTerm = EOF;
- return "";
+ return 0;
}
if( c=='"' ){
int pc, ppc;
int startLine = p->nLine;
pc = ppc = 0;
@@ -432,10 +434,38 @@
return 0;
}
return -1;
}
+/* Check to see if the string is of the form: "TAG = BOOLEAN" or just "TAG".
+** If it is, set *pValue to be the value of the boolean ("true" if there is
+** not "= BOOLEAN" component) and return non-zero. If the input string
+** does not begin with TAG, return zero.
+*/
+static int csv_boolean_parameter(
+ const char *zTag, /* Tag we are looking for */
+ int nTag, /* Size of the tag in bytes */
+ const char *z, /* Input parameter */
+ int *pValue /* Write boolean value here */
+){
+ int b;
+ z = csv_skip_whitespace(z);
+ if( strncmp(zTag, z, nTag)!=0 ) return 0;
+ z = csv_skip_whitespace(z + nTag);
+ if( z[0]==0 ){
+ *pValue = 1;
+ return 1;
+ }
+ if( z[0]!='=' ) return 0;
+ z = csv_skip_whitespace(z+1);
+ b = csv_boolean(z);
+ if( b>=0 ){
+ *pValue = b;
+ return 1;
+ }
+ return 0;
+}
/*
** Parameters:
** filename=FILENAME Name of file containing CSV content
** data=TEXT Direct CSV content.
@@ -465,10 +495,11 @@
int rc = SQLITE_OK; /* Result code from this routine */
int i, j; /* Loop counters */
#ifdef SQLITE_TEST
int tstFlags = 0; /* Value for testflags=N parameter */
#endif
+ int b; /* Value of a boolean parameter */
int nCol = -99; /* Value of the columns= parameter */
CsvReader sRdr; /* A CSV file reader used to store an error
** message and/or to count the number of columns */
static const char *azParam[] = {
"filename", "data", "schema",
@@ -489,25 +520,16 @@
if( csv_string_parameter(&sRdr, azParam[j], z, &azPValue[j]) ) break;
}
if( j=0 ){
csv_errmsg(&sRdr, "more than one 'header' parameter");
goto csvtab_connect_error;
}
- x = csv_boolean(zValue);
- if( x==1 ){
- bHeader = 1;
- }else if( x==0 ){
- bHeader = 0;
- }else{
- csv_errmsg(&sRdr, "unrecognized argument to 'header': %s", zValue);
- goto csvtab_connect_error;
- }
+ bHeader = b;
}else
#ifdef SQLITE_TEST
if( (zValue = csv_parameter("testflags",9,z))!=0 ){
tstFlags = (unsigned int)atoi(zValue);
}else
@@ -517,58 +539,98 @@
csv_errmsg(&sRdr, "more than one 'columns' parameter");
goto csvtab_connect_error;
}
nCol = atoi(zValue);
if( nCol<=0 ){
- csv_errmsg(&sRdr, "must have at least one column");
+ csv_errmsg(&sRdr, "column= value must be positive");
goto csvtab_connect_error;
}
}else
{
- csv_errmsg(&sRdr, "unrecognized parameter '%s'", z);
+ csv_errmsg(&sRdr, "bad parameter: '%s'", z);
goto csvtab_connect_error;
}
}
if( (CSV_FILENAME==0)==(CSV_DATA==0) ){
- csv_errmsg(&sRdr, "must either filename= or data= but not both");
+ csv_errmsg(&sRdr, "must specify either filename= or data= but not both");
goto csvtab_connect_error;
}
- if( nCol<=0 && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) ){
+
+ if( (nCol<=0 || bHeader==1)
+ && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA)
+ ){
goto csvtab_connect_error;
}
pNew = sqlite3_malloc( sizeof(*pNew) );
*ppVtab = (sqlite3_vtab*)pNew;
if( pNew==0 ) goto csvtab_connect_oom;
memset(pNew, 0, sizeof(*pNew));
- if( nCol>0 ){
+ if( CSV_SCHEMA==0 ){
+ sqlite3_str *pStr = sqlite3_str_new(0);
+ char *zSep = "";
+ int iCol = 0;
+ sqlite3_str_appendf(pStr, "CREATE TABLE x(");
+ if( nCol<0 && bHeader<1 ){
+ nCol = 0;
+ do{
+ csv_read_one_field(&sRdr);
+ nCol++;
+ }while( sRdr.cTerm==',' );
+ }
+ if( nCol>0 && bHeader<1 ){
+ for(iCol=0; iCol0 && iColnCol = nCol;
- }else{
+ sqlite3_str_appendf(pStr, ")");
+ CSV_SCHEMA = sqlite3_str_finish(pStr);
+ if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
+ }else if( nCol<0 ){
do{
- const char *z = csv_read_one_field(&sRdr);
- if( z==0 ) goto csvtab_connect_oom;
+ csv_read_one_field(&sRdr);
pNew->nCol++;
}while( sRdr.cTerm==',' );
+ }else{
+ pNew->nCol = nCol;
}
pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0;
pNew->zData = CSV_DATA; CSV_DATA = 0;
#ifdef SQLITE_TEST
pNew->tstFlags = tstFlags;
#endif
- pNew->iStart = bHeader==1 ? ftell(sRdr.in) : 0;
- csv_reader_reset(&sRdr);
- if( CSV_SCHEMA==0 ){
- char *zSep = "";
- CSV_SCHEMA = sqlite3_mprintf("CREATE TABLE x(");
- if( CSV_SCHEMA==0 ) goto csvtab_connect_oom;
- for(i=0; inCol; i++){
- CSV_SCHEMA = sqlite3_mprintf("%z%sc%d TEXT",CSV_SCHEMA, zSep, i);
- zSep = ",";
- }
- CSV_SCHEMA = sqlite3_mprintf("%z);", CSV_SCHEMA);
- }
+ if( bHeader!=1 ){
+ pNew->iStart = 0;
+ }else if( pNew->zData ){
+ pNew->iStart = (int)sRdr.iIn;
+ }else{
+ pNew->iStart = ftell(sRdr.in);
+ }
+ csv_reader_reset(&sRdr);
rc = sqlite3_declare_vtab(db, CSV_SCHEMA);
- if( rc ) goto csvtab_connect_error;
+ if( rc ){
+ csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db));
+ goto csvtab_connect_error;
+ }
for(i=0; irdr);
if( z==0 ){
- csv_xfer_error(pTab, &pCur->rdr);
break;
}
if( inCol ){
if( pCur->aLen[i] < pCur->rdr.n+1 ){
char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1);
Index: ext/misc/dbdump.c
==================================================================
--- ext/misc/dbdump.c
+++ ext/misc/dbdump.c
@@ -139,49 +139,16 @@
** that quoting is required.
**
** Return '"' if quoting is required. Return 0 if no quoting is required.
*/
static char quoteChar(const char *zName){
- /* All SQLite keywords, in alphabetical order */
- static const char *azKeywords[] = {
- "ABORT", "ACTION", "ADD", "AFTER", "ALL", "ALTER", "ANALYZE", "AND", "AS",
- "ASC", "ATTACH", "AUTOINCREMENT", "BEFORE", "BEGIN", "BETWEEN", "BY",
- "CASCADE", "CASE", "CAST", "CHECK", "COLLATE", "COLUMN", "COMMIT",
- "CONFLICT", "CONSTRAINT", "CREATE", "CROSS", "CURRENT_DATE",
- "CURRENT_TIME", "CURRENT_TIMESTAMP", "DATABASE", "DEFAULT", "DEFERRABLE",
- "DEFERRED", "DELETE", "DESC", "DETACH", "DISTINCT", "DROP", "EACH",
- "ELSE", "END", "ESCAPE", "EXCEPT", "EXCLUSIVE", "EXISTS", "EXPLAIN",
- "FAIL", "FOR", "FOREIGN", "FROM", "FULL", "GLOB", "GROUP", "HAVING", "IF",
- "IGNORE", "IMMEDIATE", "IN", "INDEX", "INDEXED", "INITIALLY", "INNER",
- "INSERT", "INSTEAD", "INTERSECT", "INTO", "IS", "ISNULL", "JOIN", "KEY",
- "LEFT", "LIKE", "LIMIT", "MATCH", "NATURAL", "NO", "NOT", "NOTNULL",
- "NULL", "OF", "OFFSET", "ON", "OR", "ORDER", "OUTER", "PLAN", "PRAGMA",
- "PRIMARY", "QUERY", "RAISE", "RECURSIVE", "REFERENCES", "REGEXP",
- "REINDEX", "RELEASE", "RENAME", "REPLACE", "RESTRICT", "RIGHT",
- "ROLLBACK", "ROW", "SAVEPOINT", "SELECT", "SET", "TABLE", "TEMP",
- "TEMPORARY", "THEN", "TO", "TRANSACTION", "TRIGGER", "UNION", "UNIQUE",
- "UPDATE", "USING", "VACUUM", "VALUES", "VIEW", "VIRTUAL", "WHEN", "WHERE",
- "WITH", "WITHOUT",
- };
- int i, lwr, upr, mid, c;
+ int i;
if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"';
for(i=0; zName[i]; i++){
if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"';
}
- lwr = 0;
- upr = sizeof(azKeywords)/sizeof(azKeywords[0]) - 1;
- while( lwr<=upr ){
- mid = (lwr+upr)/2;
- c = sqlite3_stricmp(azKeywords[mid], zName);
- if( c==0 ) return '"';
- if( c<0 ){
- lwr = mid+1;
- }else{
- upr = mid-1;
- }
- }
- return 0;
+ return sqlite3_keyword_check(zName, i) ? '"' : 0;
}
/*
** Release memory previously allocated by tableColumnList().
@@ -291,11 +258,10 @@
if( i>nCol ){
/* At this point, we know that azRowid[j] is not the name of any
** ordinary column in the table. Verify that azRowid[j] is a valid
** name for the rowid before adding it to azCol[0]. WITHOUT ROWID
** tables will fail this last check */
- int rc;
rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0);
if( rc==SQLITE_OK ) azCol[0] = azRowid[j];
break;
}
}
@@ -453,16 +419,16 @@
}
if( strcmp(zType, "table")==0 ){
DText sSelect;
DText sTable;
- char **azCol;
+ char **azTCol;
int i;
int nCol;
- azCol = tableColumnList(p, zTable);
- if( azCol==0 ) return 0;
+ azTCol = tableColumnList(p, zTable);
+ if( azTCol==0 ) return 0;
initText(&sTable);
appendText(&sTable, "INSERT INTO ", 0);
/* Always quote the table name, even if it appears to be pure ascii,
@@ -471,37 +437,37 @@
/* If preserving the rowid, add a column list after the table name.
** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)"
** instead of the usual "INSERT INTO tab VALUES(...)".
*/
- if( azCol[0] ){
+ if( azTCol[0] ){
appendText(&sTable, "(", 0);
- appendText(&sTable, azCol[0], 0);
- for(i=1; azCol[i]; i++){
+ appendText(&sTable, azTCol[0], 0);
+ for(i=1; azTCol[i]; i++){
appendText(&sTable, ",", 0);
- appendText(&sTable, azCol[i], quoteChar(azCol[i]));
+ appendText(&sTable, azTCol[i], quoteChar(azTCol[i]));
}
appendText(&sTable, ")", 0);
}
appendText(&sTable, " VALUES(", 0);
/* Build an appropriate SELECT statement */
initText(&sSelect);
appendText(&sSelect, "SELECT ", 0);
- if( azCol[0] ){
- appendText(&sSelect, azCol[0], 0);
+ if( azTCol[0] ){
+ appendText(&sSelect, azTCol[0], 0);
appendText(&sSelect, ",", 0);
}
- for(i=1; azCol[i]; i++){
- appendText(&sSelect, azCol[i], quoteChar(azCol[i]));
- if( azCol[i+1] ){
+ for(i=1; azTCol[i]; i++){
+ appendText(&sSelect, azTCol[i], quoteChar(azTCol[i]));
+ if( azTCol[i+1] ){
appendText(&sSelect, ",", 0);
}
}
nCol = i;
- if( azCol[0]==0 ) nCol--;
- freeColumnList(azCol);
+ if( azTCol[0]==0 ) nCol--;
+ freeColumnList(azTCol);
appendText(&sSelect, " FROM ", 0);
appendText(&sSelect, zTable, quoteChar(zTable));
rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0);
if( rc!=SQLITE_OK ){
@@ -517,11 +483,19 @@
output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i));
break;
}
case SQLITE_FLOAT: {
double r = sqlite3_column_double(pStmt,i);
- output_formatted(p, "%!.20g", r);
+ sqlite3_uint64 ur;
+ memcpy(&ur,&r,sizeof(r));
+ if( ur==0x7ff0000000000000LL ){
+ p->xCallback("1e999", p->pArg);
+ }else if( ur==0xfff0000000000000LL ){
+ p->xCallback("-1e999", p->pArg);
+ }else{
+ output_formatted(p, "%!.20g", r);
+ }
break;
}
case SQLITE_NULL: {
p->xCallback("NULL", p->pArg);
break;
Index: ext/misc/eval.c
==================================================================
--- ext/misc/eval.c
+++ ext/misc/eval.c
@@ -32,10 +32,11 @@
** Callback from sqlite_exec() for the eval() function.
*/
static int callback(void *pCtx, int argc, char **argv, char **colnames){
struct EvalResult *p = (struct EvalResult*)pCtx;
int i;
+ if( argv==0 ) return 0;
for(i=0; inUsed+p->szSep+1 > p->nAlloc ){
char *zNew;
ADDED ext/misc/explain.c
Index: ext/misc/explain.c
==================================================================
--- /dev/null
+++ ext/misc/explain.c
@@ -0,0 +1,322 @@
+/*
+** 2018-09-16
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file demonstrates an eponymous virtual table that returns the
+** EXPLAIN output from an SQL statement.
+**
+** Usage example:
+**
+** .load ./explain
+** SELECT p2 FROM explain('SELECT * FROM sqlite_master')
+** WHERE opcode='OpenRead';
+**
+** This module was originally written to help simplify SQLite testing,
+** by providing an easier means of verifying certain patterns in the
+** generated bytecode.
+*/
+#if !defined(SQLITEINT_H)
+#include "sqlite3ext.h"
+#endif
+SQLITE_EXTENSION_INIT1
+#include
+#include
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* explain_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a explain virtual table
+*/
+typedef struct explain_vtab explain_vtab;
+struct explain_vtab {
+ sqlite3_vtab base; /* Base class - must be first */
+ sqlite3 *db; /* Database connection for this explain vtab */
+};
+
+/* explain_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result from an EXPLAIN operation.
+*/
+typedef struct explain_cursor explain_cursor;
+struct explain_cursor {
+ sqlite3_vtab_cursor base; /* Base class - must be first */
+ sqlite3 *db; /* Database connection for this cursor */
+ char *zSql; /* Value for the EXPLN_COLUMN_SQL column */
+ sqlite3_stmt *pExplain; /* Statement being explained */
+ int rc; /* Result of last sqlite3_step() on pExplain */
+};
+
+/*
+** The explainConnect() method is invoked to create a new
+** explain_vtab that describes the explain virtual table.
+**
+** Think of this routine as the constructor for explain_vtab objects.
+**
+** All this routine needs to do is:
+**
+** (1) Allocate the explain_vtab object and initialize all fields.
+**
+** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+** result set of queries against explain will look like.
+*/
+static int explainConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ explain_vtab *pNew;
+ int rc;
+
+/* Column numbers */
+#define EXPLN_COLUMN_ADDR 0 /* Instruction address */
+#define EXPLN_COLUMN_OPCODE 1 /* Opcode */
+#define EXPLN_COLUMN_P1 2 /* Operand 1 */
+#define EXPLN_COLUMN_P2 3 /* Operand 2 */
+#define EXPLN_COLUMN_P3 4 /* Operand 3 */
+#define EXPLN_COLUMN_P4 5 /* Operand 4 */
+#define EXPLN_COLUMN_P5 6 /* Operand 5 */
+#define EXPLN_COLUMN_COMMENT 7 /* Comment */
+#define EXPLN_COLUMN_SQL 8 /* SQL that is being explained */
+
+
+ rc = sqlite3_declare_vtab(db,
+ "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)");
+ if( rc==SQLITE_OK ){
+ pNew = sqlite3_malloc( sizeof(*pNew) );
+ *ppVtab = (sqlite3_vtab*)pNew;
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(*pNew));
+ pNew->db = db;
+ }
+ return rc;
+}
+
+/*
+** This method is the destructor for explain_cursor objects.
+*/
+static int explainDisconnect(sqlite3_vtab *pVtab){
+ sqlite3_free(pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new explain_cursor object.
+*/
+static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+ explain_cursor *pCur;
+ pCur = sqlite3_malloc( sizeof(*pCur) );
+ if( pCur==0 ) return SQLITE_NOMEM;
+ memset(pCur, 0, sizeof(*pCur));
+ pCur->db = ((explain_vtab*)p)->db;
+ *ppCursor = &pCur->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destructor for a explain_cursor.
+*/
+static int explainClose(sqlite3_vtab_cursor *cur){
+ explain_cursor *pCur = (explain_cursor*)cur;
+ sqlite3_finalize(pCur->pExplain);
+ sqlite3_free(pCur->zSql);
+ sqlite3_free(pCur);
+ return SQLITE_OK;
+}
+
+
+/*
+** Advance a explain_cursor to its next row of output.
+*/
+static int explainNext(sqlite3_vtab_cursor *cur){
+ explain_cursor *pCur = (explain_cursor*)cur;
+ pCur->rc = sqlite3_step(pCur->pExplain);
+ if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc;
+ return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the explain_cursor
+** is currently pointing.
+*/
+static int explainColumn(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int i /* Which column to return */
+){
+ explain_cursor *pCur = (explain_cursor*)cur;
+ if( i==EXPLN_COLUMN_SQL ){
+ sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i));
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** rowid is the same as the output value.
+*/
+static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+ explain_cursor *pCur = (explain_cursor*)cur;
+ *pRowid = sqlite3_column_int64(pCur->pExplain, 0);
+ return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int explainEof(sqlite3_vtab_cursor *cur){
+ explain_cursor *pCur = (explain_cursor*)cur;
+ return pCur->rc!=SQLITE_ROW;
+}
+
+/*
+** This method is called to "rewind" the explain_cursor object back
+** to the first row of output. This method is always called at least
+** once prior to any call to explainColumn() or explainRowid() or
+** explainEof().
+**
+** The argv[0] is the SQL statement that is to be explained.
+*/
+static int explainFilter(
+ sqlite3_vtab_cursor *pVtabCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ explain_cursor *pCur = (explain_cursor *)pVtabCursor;
+ char *zSql = 0;
+ int rc;
+ sqlite3_finalize(pCur->pExplain);
+ pCur->pExplain = 0;
+ if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){
+ pCur->rc = SQLITE_DONE;
+ return SQLITE_OK;
+ }
+ sqlite3_free(pCur->zSql);
+ pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0]));
+ if( pCur->zSql ){
+ zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql);
+ }
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0);
+ sqlite3_free(zSql);
+ }
+ if( rc ){
+ sqlite3_finalize(pCur->pExplain);
+ pCur->pExplain = 0;
+ sqlite3_free(pCur->zSql);
+ pCur->zSql = 0;
+ }else{
+ pCur->rc = sqlite3_step(pCur->pExplain);
+ rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc;
+ }
+ return rc;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the explain virtual table. This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int explainBestIndex(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ int i; /* Loop counter */
+ int idx = -1; /* Index of a usable == constraint against SQL */
+ int unusable = 0; /* True if there are unusable constraints on SQL */
+
+ pIdxInfo->estimatedRows = 500;
+ for(i=0; inConstraint; i++){
+ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i];
+ if( p->iColumn!=EXPLN_COLUMN_SQL ) continue;
+ if( !p->usable ){
+ unusable = 1;
+ }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ idx = i;
+ }
+ }
+ if( idx>=0 ){
+ /* There exists a usable == constraint against the SQL column */
+ pIdxInfo->estimatedCost = 10.0;
+ pIdxInfo->idxNum = 1;
+ pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[idx].omit = 1;
+ }else if( unusable ){
+ /* There are unusable constraints against the SQL column. Do not allow
+ ** this plan to continue forward. */
+ return SQLITE_CONSTRAINT;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the
+** explain virtual table.
+*/
+static sqlite3_module explainModule = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ explainConnect, /* xConnect */
+ explainBestIndex, /* xBestIndex */
+ explainDisconnect, /* xDisconnect */
+ 0, /* xDestroy */
+ explainOpen, /* xOpen - open a cursor */
+ explainClose, /* xClose - close a cursor */
+ explainFilter, /* xFilter - configure scan constraints */
+ explainNext, /* xNext - advance a cursor */
+ explainEof, /* xEof - check for end of scan */
+ explainColumn, /* xColumn - read data */
+ explainRowid, /* xRowid - read data */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindMethod */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0, /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3ExplainVtabInit(sqlite3 *db){
+ int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ rc = sqlite3_create_module(db, "explain", &explainModule, 0);
+#endif
+ return rc;
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_explain_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ int rc = SQLITE_OK;
+ SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ rc = sqlite3ExplainVtabInit(db);
+#endif
+ return rc;
+}
Index: ext/misc/fileio.c
==================================================================
--- ext/misc/fileio.c
+++ ext/misc/fileio.c
@@ -9,15 +9,140 @@
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This SQLite extension implements SQL functions readfile() and
-** writefile().
+** writefile(), and eponymous virtual type "fsdir".
+**
+** WRITEFILE(FILE, DATA [, MODE [, MTIME]]):
+**
+** If neither of the optional arguments is present, then this UDF
+** function writes blob DATA to file FILE. If successful, the number
+** of bytes written is returned. If an error occurs, NULL is returned.
+**
+** If the first option argument - MODE - is present, then it must
+** be passed an integer value that corresponds to a POSIX mode
+** value (file type + permissions, as returned in the stat.st_mode
+** field by the stat() system call). Three types of files may
+** be written/created:
+**
+** regular files: (mode & 0170000)==0100000
+** symbolic links: (mode & 0170000)==0120000
+** directories: (mode & 0170000)==0040000
+**
+** For a directory, the DATA is ignored. For a symbolic link, it is
+** interpreted as text and used as the target of the link. For a
+** regular file, it is interpreted as a blob and written into the
+** named file. Regardless of the type of file, its permissions are
+** set to (mode & 0777) before returning.
+**
+** If the optional MTIME argument is present, then it is interpreted
+** as an integer - the number of seconds since the unix epoch. The
+** modification-time of the target file is set to this value before
+** returning.
+**
+** If three or more arguments are passed to this function and an
+** error is encountered, an exception is raised.
+**
+** READFILE(FILE):
+**
+** Read and return the contents of file FILE (type blob) from disk.
+**
+** FSDIR:
+**
+** Used as follows:
+**
+** SELECT * FROM fsdir($path [, $dir]);
+**
+** Parameter $path is an absolute or relative pathname. If the file that it
+** refers to does not exist, it is an error. If the path refers to a regular
+** file or symbolic link, it returns a single row. Or, if the path refers
+** to a directory, it returns one row for the directory, and one row for each
+** file within the hierarchy rooted at $path.
+**
+** Each row has the following columns:
+**
+** name: Path to file or directory (text value).
+** mode: Value of stat.st_mode for directory entry (an integer).
+** mtime: Value of stat.st_mtime for directory entry (an integer).
+** data: For a regular file, a blob containing the file data. For a
+** symlink, a text value containing the text of the link. For a
+** directory, NULL.
+**
+** If a non-NULL value is specified for the optional $dir parameter and
+** $path is a relative path, then $path is interpreted relative to $dir.
+** And the paths returned in the "name" column of the table are also
+** relative to directory $dir.
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include
+#include
+#include
+
+#include
+#include
+#include
+#if !defined(_WIN32) && !defined(WIN32)
+# include
+# include
+# include
+# include
+#else
+# include "windows.h"
+# include
+# include
+# include "test_windirent.h"
+# define dirent DIRENT
+# ifndef chmod
+# define chmod _chmod
+# endif
+# ifndef stat
+# define stat _stat
+# endif
+# define mkdir(path,mode) _mkdir(path)
+# define lstat(path,buf) stat(path,buf)
+#endif
+#include
+#include
+
+
+/*
+** Structure of the fsdir() table-valued function
+*/
+ /* 0 1 2 3 4 5 */
+#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)"
+#define FSDIR_COLUMN_NAME 0 /* Name of the file */
+#define FSDIR_COLUMN_MODE 1 /* Access mode */
+#define FSDIR_COLUMN_MTIME 2 /* Last modification time */
+#define FSDIR_COLUMN_DATA 3 /* File content */
+#define FSDIR_COLUMN_PATH 4 /* Path to top of search */
+#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */
+
+
+/*
+** Set the result stored by context ctx to a blob containing the
+** contents of file zName.
+*/
+static void readFileContents(sqlite3_context *ctx, const char *zName){
+ FILE *in;
+ long nIn;
+ void *pBuf;
+
+ in = fopen(zName, "rb");
+ if( in==0 ) return;
+ fseek(in, 0, SEEK_END);
+ nIn = ftell(in);
+ rewind(in);
+ pBuf = sqlite3_malloc( nIn );
+ if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+ sqlite3_result_blob(ctx, pBuf, nIn, sqlite3_free);
+ }else{
+ sqlite3_free(pBuf);
+ }
+ fclose(in);
+}
/*
** Implementation of the "readfile(X)" SQL function. The entire content
** of the file named X is read and returned as a BLOB. NULL is returned
** if the file does not exist or is unreadable.
@@ -26,57 +151,787 @@
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
const char *zName;
- FILE *in;
- long nIn;
- void *pBuf;
-
(void)(argc); /* Unused parameter */
zName = (const char*)sqlite3_value_text(argv[0]);
if( zName==0 ) return;
- in = fopen(zName, "rb");
- if( in==0 ) return;
- fseek(in, 0, SEEK_END);
- nIn = ftell(in);
- rewind(in);
- pBuf = sqlite3_malloc( nIn );
- if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
- sqlite3_result_blob(context, pBuf, nIn, sqlite3_free);
+ readFileContents(context, zName);
+}
+
+/*
+** Set the error message contained in context ctx to the results of
+** vprintf(zFmt, ...).
+*/
+static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+ char *zMsg = 0;
+ va_list ap;
+ va_start(ap, zFmt);
+ zMsg = sqlite3_vmprintf(zFmt, ap);
+ sqlite3_result_error(ctx, zMsg, -1);
+ sqlite3_free(zMsg);
+ va_end(ap);
+}
+
+#if defined(_WIN32)
+/*
+** This function is designed to convert a Win32 FILETIME structure into the
+** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
+*/
+static sqlite3_uint64 fileTimeToUnixTime(
+ LPFILETIME pFileTime
+){
+ SYSTEMTIME epochSystemTime;
+ ULARGE_INTEGER epochIntervals;
+ FILETIME epochFileTime;
+ ULARGE_INTEGER fileIntervals;
+
+ memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
+ epochSystemTime.wYear = 1970;
+ epochSystemTime.wMonth = 1;
+ epochSystemTime.wDay = 1;
+ SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
+ epochIntervals.LowPart = epochFileTime.dwLowDateTime;
+ epochIntervals.HighPart = epochFileTime.dwHighDateTime;
+
+ fileIntervals.LowPart = pFileTime->dwLowDateTime;
+ fileIntervals.HighPart = pFileTime->dwHighDateTime;
+
+ return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
+}
+
+/*
+** This function attempts to normalize the time values found in the stat()
+** buffer to UTC. This is necessary on Win32, where the runtime library
+** appears to return these values as local times.
+*/
+static void statTimesToUtc(
+ const char *zPath,
+ struct stat *pStatBuf
+){
+ HANDLE hFindFile;
+ WIN32_FIND_DATAW fd;
+ LPWSTR zUnicodeName;
+ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+ zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
+ if( zUnicodeName ){
+ memset(&fd, 0, sizeof(WIN32_FIND_DATAW));
+ hFindFile = FindFirstFileW(zUnicodeName, &fd);
+ if( hFindFile!=NULL ){
+ pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
+ pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
+ pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
+ FindClose(hFindFile);
+ }
+ sqlite3_free(zUnicodeName);
+ }
+}
+#endif
+
+/*
+** This function is used in place of stat(). On Windows, special handling
+** is required in order for the included time to be returned as UTC. On all
+** other systems, this function simply calls stat().
+*/
+static int fileStat(
+ const char *zPath,
+ struct stat *pStatBuf
+){
+#if defined(_WIN32)
+ int rc = stat(zPath, pStatBuf);
+ if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+ return rc;
+#else
+ return stat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** This function is used in place of lstat(). On Windows, special handling
+** is required in order for the included time to be returned as UTC. On all
+** other systems, this function simply calls lstat().
+*/
+static int fileLinkStat(
+ const char *zPath,
+ struct stat *pStatBuf
+){
+#if defined(_WIN32)
+ int rc = lstat(zPath, pStatBuf);
+ if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
+ return rc;
+#else
+ return lstat(zPath, pStatBuf);
+#endif
+}
+
+/*
+** Argument zFile is the name of a file that will be created and/or written
+** by SQL function writefile(). This function ensures that the directory
+** zFile will be written to exists, creating it if required. The permissions
+** for any path components created by this function are set to (mode&0777).
+**
+** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise,
+** SQLITE_OK is returned if the directory is successfully created, or
+** SQLITE_ERROR otherwise.
+*/
+static int makeDirectory(
+ const char *zFile,
+ mode_t mode
+){
+ char *zCopy = sqlite3_mprintf("%s", zFile);
+ int rc = SQLITE_OK;
+
+ if( zCopy==0 ){
+ rc = SQLITE_NOMEM;
}else{
- sqlite3_free(pBuf);
+ int nCopy = (int)strlen(zCopy);
+ int i = 1;
+
+ while( rc==SQLITE_OK ){
+ struct stat sStat;
+ int rc2;
+
+ for(; zCopy[i]!='/' && i=0 ){
+#if defined(_WIN32)
+ /* Windows */
+ FILETIME lastAccess;
+ FILETIME lastWrite;
+ SYSTEMTIME currentTime;
+ LONGLONG intervals;
+ HANDLE hFile;
+ LPWSTR zUnicodeName;
+ extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
+
+ GetSystemTime(¤tTime);
+ SystemTimeToFileTime(¤tTime, &lastAccess);
+ intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
+ lastWrite.dwLowDateTime = (DWORD)intervals;
+ lastWrite.dwHighDateTime = intervals >> 32;
+ zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
+ if( zUnicodeName==0 ){
+ return 1;
+ }
+ hFile = CreateFileW(
+ zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
+ FILE_FLAG_BACKUP_SEMANTICS, NULL
+ );
+ sqlite3_free(zUnicodeName);
+ if( hFile!=INVALID_HANDLE_VALUE ){
+ BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
+ CloseHandle(hFile);
+ return !bResult;
+ }else{
+ return 1;
+ }
+#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
+ /* Recent unix */
+ struct timespec times[2];
+ times[0].tv_nsec = times[1].tv_nsec = 0;
+ times[0].tv_sec = time(0);
+ times[1].tv_sec = mtime;
+ if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
+ return 1;
+ }
+#else
+ /* Legacy unix */
+ struct timeval times[2];
+ times[0].tv_usec = times[1].tv_usec = 0;
+ times[0].tv_sec = time(0);
+ times[1].tv_sec = mtime;
+ if( utimes(zFile, times) ){
+ return 1;
+ }
+#endif
}
- fclose(in);
+
+ return 0;
}
/*
-** Implementation of the "writefile(X,Y)" SQL function. The argument Y
-** is written into file X. The number of bytes written is returned. Or
-** NULL is returned if something goes wrong, such as being unable to open
-** file X for writing.
+** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function.
+** Refer to header comments at the top of this file for details.
*/
static void writefileFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
- FILE *out;
- const char *z;
- sqlite3_int64 rc;
const char *zFile;
+ mode_t mode = 0;
+ int res;
+ sqlite3_int64 mtime = -1;
- (void)(argc); /* Unused parameter */
+ if( argc<2 || argc>4 ){
+ sqlite3_result_error(context,
+ "wrong number of arguments to function writefile()", -1
+ );
+ return;
+ }
+
zFile = (const char*)sqlite3_value_text(argv[0]);
if( zFile==0 ) return;
- out = fopen(zFile, "wb");
- if( out==0 ) return;
- z = (const char*)sqlite3_value_blob(argv[1]);
- if( z==0 ){
- rc = 0;
+ if( argc>=3 ){
+ mode = (mode_t)sqlite3_value_int(argv[2]);
+ }
+ if( argc==4 ){
+ mtime = sqlite3_value_int64(argv[3]);
+ }
+
+ res = writeFile(context, zFile, argv[1], mode, mtime);
+ if( res==1 && errno==ENOENT ){
+ if( makeDirectory(zFile, mode)==SQLITE_OK ){
+ res = writeFile(context, zFile, argv[1], mode, mtime);
+ }
+ }
+
+ if( argc>2 && res!=0 ){
+ if( S_ISLNK(mode) ){
+ ctxErrorMsg(context, "failed to create symlink: %s", zFile);
+ }else if( S_ISDIR(mode) ){
+ ctxErrorMsg(context, "failed to create directory: %s", zFile);
+ }else{
+ ctxErrorMsg(context, "failed to write file: %s", zFile);
+ }
+ }
+}
+
+/*
+** SQL function: lsmode(MODE)
+**
+** Given a numberic st_mode from stat(), convert it into a human-readable
+** text string in the style of "ls -l".
+*/
+static void lsModeFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ int i;
+ int iMode = sqlite3_value_int(argv[0]);
+ char z[16];
+ (void)argc;
+ if( S_ISLNK(iMode) ){
+ z[0] = 'l';
+ }else if( S_ISREG(iMode) ){
+ z[0] = '-';
+ }else if( S_ISDIR(iMode) ){
+ z[0] = 'd';
+ }else{
+ z[0] = '?';
+ }
+ for(i=0; i<3; i++){
+ int m = (iMode >> ((2-i)*3));
+ char *a = &z[1 + i*3];
+ a[0] = (m & 0x4) ? 'r' : '-';
+ a[1] = (m & 0x2) ? 'w' : '-';
+ a[2] = (m & 0x1) ? 'x' : '-';
+ }
+ z[10] = '\0';
+ sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT);
+}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Cursor type for recursively iterating through a directory structure.
+*/
+typedef struct fsdir_cursor fsdir_cursor;
+typedef struct FsdirLevel FsdirLevel;
+
+struct FsdirLevel {
+ DIR *pDir; /* From opendir() */
+ char *zDir; /* Name of directory (nul-terminated) */
+};
+
+struct fsdir_cursor {
+ sqlite3_vtab_cursor base; /* Base class - must be first */
+
+ int nLvl; /* Number of entries in aLvl[] array */
+ int iLvl; /* Index of current entry */
+ FsdirLevel *aLvl; /* Hierarchy of directories being traversed */
+
+ const char *zBase;
+ int nBase;
+
+ struct stat sStat; /* Current lstat() results */
+ char *zPath; /* Path to current entry */
+ sqlite3_int64 iRowid; /* Current rowid */
+};
+
+typedef struct fsdir_tab fsdir_tab;
+struct fsdir_tab {
+ sqlite3_vtab base; /* Base class - must be first */
+};
+
+/*
+** Construct a new fsdir virtual table object.
+*/
+static int fsdirConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ fsdir_tab *pNew = 0;
+ int rc;
+ (void)pAux;
+ (void)argc;
+ (void)argv;
+ (void)pzErr;
+ rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA);
+ if( rc==SQLITE_OK ){
+ pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) );
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(*pNew));
+ }
+ *ppVtab = (sqlite3_vtab*)pNew;
+ return rc;
+}
+
+/*
+** This method is the destructor for fsdir vtab objects.
+*/
+static int fsdirDisconnect(sqlite3_vtab *pVtab){
+ sqlite3_free(pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new fsdir_cursor object.
+*/
+static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+ fsdir_cursor *pCur;
+ (void)p;
+ pCur = sqlite3_malloc( sizeof(*pCur) );
+ if( pCur==0 ) return SQLITE_NOMEM;
+ memset(pCur, 0, sizeof(*pCur));
+ pCur->iLvl = -1;
+ *ppCursor = &pCur->base;
+ return SQLITE_OK;
+}
+
+/*
+** Reset a cursor back to the state it was in when first returned
+** by fsdirOpen().
+*/
+static void fsdirResetCursor(fsdir_cursor *pCur){
+ int i;
+ for(i=0; i<=pCur->iLvl; i++){
+ FsdirLevel *pLvl = &pCur->aLvl[i];
+ if( pLvl->pDir ) closedir(pLvl->pDir);
+ sqlite3_free(pLvl->zDir);
+ }
+ sqlite3_free(pCur->zPath);
+ sqlite3_free(pCur->aLvl);
+ pCur->aLvl = 0;
+ pCur->zPath = 0;
+ pCur->zBase = 0;
+ pCur->nBase = 0;
+ pCur->nLvl = 0;
+ pCur->iLvl = -1;
+ pCur->iRowid = 1;
+}
+
+/*
+** Destructor for an fsdir_cursor.
+*/
+static int fsdirClose(sqlite3_vtab_cursor *cur){
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+
+ fsdirResetCursor(pCur);
+ sqlite3_free(pCur);
+ return SQLITE_OK;
+}
+
+/*
+** Set the error message for the virtual table associated with cursor
+** pCur to the results of vprintf(zFmt, ...).
+*/
+static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){
+ va_list ap;
+ va_start(ap, zFmt);
+ pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+}
+
+
+/*
+** Advance an fsdir_cursor to its next row of output.
+*/
+static int fsdirNext(sqlite3_vtab_cursor *cur){
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+ mode_t m = pCur->sStat.st_mode;
+
+ pCur->iRowid++;
+ if( S_ISDIR(m) ){
+ /* Descend into this directory */
+ int iNew = pCur->iLvl + 1;
+ FsdirLevel *pLvl;
+ if( iNew>=pCur->nLvl ){
+ int nNew = iNew+1;
+ int nByte = nNew*sizeof(FsdirLevel);
+ FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc(pCur->aLvl, nByte);
+ if( aNew==0 ) return SQLITE_NOMEM;
+ memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl));
+ pCur->aLvl = aNew;
+ pCur->nLvl = nNew;
+ }
+ pCur->iLvl = iNew;
+ pLvl = &pCur->aLvl[iNew];
+
+ pLvl->zDir = pCur->zPath;
+ pCur->zPath = 0;
+ pLvl->pDir = opendir(pLvl->zDir);
+ if( pLvl->pDir==0 ){
+ fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath);
+ return SQLITE_ERROR;
+ }
+ }
+
+ while( pCur->iLvl>=0 ){
+ FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl];
+ struct dirent *pEntry = readdir(pLvl->pDir);
+ if( pEntry ){
+ if( pEntry->d_name[0]=='.' ){
+ if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
+ if( pEntry->d_name[1]=='\0' ) continue;
+ }
+ sqlite3_free(pCur->zPath);
+ pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
+ if( pCur->zPath==0 ) return SQLITE_NOMEM;
+ if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+ fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+ return SQLITE_ERROR;
+ }
+ return SQLITE_OK;
+ }
+ closedir(pLvl->pDir);
+ sqlite3_free(pLvl->zDir);
+ pLvl->pDir = 0;
+ pLvl->zDir = 0;
+ pCur->iLvl--;
+ }
+
+ /* EOF */
+ sqlite3_free(pCur->zPath);
+ pCur->zPath = 0;
+ return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int fsdirColumn(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int i /* Which column to return */
+){
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+ switch( i ){
+ case FSDIR_COLUMN_NAME: {
+ sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT);
+ break;
+ }
+
+ case FSDIR_COLUMN_MODE:
+ sqlite3_result_int64(ctx, pCur->sStat.st_mode);
+ break;
+
+ case FSDIR_COLUMN_MTIME:
+ sqlite3_result_int64(ctx, pCur->sStat.st_mtime);
+ break;
+
+ case FSDIR_COLUMN_DATA: {
+ mode_t m = pCur->sStat.st_mode;
+ if( S_ISDIR(m) ){
+ sqlite3_result_null(ctx);
+#if !defined(_WIN32) && !defined(WIN32)
+ }else if( S_ISLNK(m) ){
+ char aStatic[64];
+ char *aBuf = aStatic;
+ int nBuf = 64;
+ int n;
+
+ while( 1 ){
+ n = readlink(pCur->zPath, aBuf, nBuf);
+ if( nzPath);
+ }
+ }
+ case FSDIR_COLUMN_PATH:
+ default: {
+ /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters.
+ ** always return their values as NULL */
+ break;
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** first row returned is assigned rowid value 1, and each subsequent
+** row a value 1 more than that of the previous.
+*/
+static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+ *pRowid = pCur->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int fsdirEof(sqlite3_vtab_cursor *cur){
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+ return (pCur->zPath==0);
+}
+
+/*
+** xFilter callback.
+**
+** idxNum==1 PATH parameter only
+** idxNum==2 Both PATH and DIR supplied
+*/
+static int fsdirFilter(
+ sqlite3_vtab_cursor *cur,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ const char *zDir = 0;
+ fsdir_cursor *pCur = (fsdir_cursor*)cur;
+ (void)idxStr;
+ fsdirResetCursor(pCur);
+
+ if( idxNum==0 ){
+ fsdirSetErrmsg(pCur, "table function fsdir requires an argument");
+ return SQLITE_ERROR;
+ }
+
+ assert( argc==idxNum && (argc==1 || argc==2) );
+ zDir = (const char*)sqlite3_value_text(argv[0]);
+ if( zDir==0 ){
+ fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument");
+ return SQLITE_ERROR;
+ }
+ if( argc==2 ){
+ pCur->zBase = (const char*)sqlite3_value_text(argv[1]);
+ }
+ if( pCur->zBase ){
+ pCur->nBase = (int)strlen(pCur->zBase)+1;
+ pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir);
+ }else{
+ pCur->zPath = sqlite3_mprintf("%s", zDir);
+ }
+
+ if( pCur->zPath==0 ){
+ return SQLITE_NOMEM;
+ }
+ if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
+ fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
+ return SQLITE_ERROR;
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the generate_series virtual table. This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+**
+** In this implementation idxNum is used to represent the
+** query plan. idxStr is unused.
+**
+** The query plan is represented by values of idxNum:
+**
+** (1) The path value is supplied by argv[0]
+** (2) Path is in argv[0] and dir is in argv[1]
+*/
+static int fsdirBestIndex(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ int i; /* Loop over constraints */
+ int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */
+ int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */
+ int seenPath = 0; /* True if an unusable PATH= constraint is seen */
+ int seenDir = 0; /* True if an unusable DIR= constraint is seen */
+ const struct sqlite3_index_constraint *pConstraint;
+
+ (void)tab;
+ pConstraint = pIdxInfo->aConstraint;
+ for(i=0; inConstraint; i++, pConstraint++){
+ if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+ switch( pConstraint->iColumn ){
+ case FSDIR_COLUMN_PATH: {
+ if( pConstraint->usable ){
+ idxPath = i;
+ seenPath = 0;
+ }else if( idxPath<0 ){
+ seenPath = 1;
+ }
+ break;
+ }
+ case FSDIR_COLUMN_DIR: {
+ if( pConstraint->usable ){
+ idxDir = i;
+ seenDir = 0;
+ }else if( idxDir<0 ){
+ seenDir = 1;
+ }
+ break;
+ }
+ }
+ }
+ if( seenPath || seenDir ){
+ /* If input parameters are unusable, disallow this plan */
+ return SQLITE_CONSTRAINT;
+ }
+
+ if( idxPath<0 ){
+ pIdxInfo->idxNum = 0;
+ /* The pIdxInfo->estimatedCost should have been initialized to a huge
+ ** number. Leave it unchanged. */
+ pIdxInfo->estimatedRows = 0x7fffffff;
}else{
- rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out);
+ pIdxInfo->aConstraintUsage[idxPath].omit = 1;
+ pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1;
+ if( idxDir>=0 ){
+ pIdxInfo->aConstraintUsage[idxDir].omit = 1;
+ pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2;
+ pIdxInfo->idxNum = 2;
+ pIdxInfo->estimatedCost = 10.0;
+ }else{
+ pIdxInfo->idxNum = 1;
+ pIdxInfo->estimatedCost = 100.0;
+ }
}
- fclose(out);
- sqlite3_result_int64(context, rc);
+
+ return SQLITE_OK;
}
+/*
+** Register the "fsdir" virtual table.
+*/
+static int fsdirRegister(sqlite3 *db){
+ static sqlite3_module fsdirModule = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ fsdirConnect, /* xConnect */
+ fsdirBestIndex, /* xBestIndex */
+ fsdirDisconnect, /* xDisconnect */
+ 0, /* xDestroy */
+ fsdirOpen, /* xOpen - open a cursor */
+ fsdirClose, /* xClose - close a cursor */
+ fsdirFilter, /* xFilter - configure scan constraints */
+ fsdirNext, /* xNext - advance a cursor */
+ fsdirEof, /* xEof - check for end of scan */
+ fsdirColumn, /* xColumn - read data */
+ fsdirRowid, /* xRowid - read data */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindMethod */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0, /* xShadowName */
+ };
+
@@ -83,5 +938,11 @@
+ int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0);
+ return rc;
+}
+#else /* SQLITE_OMIT_VIRTUALTABLE */
+# define fsdirRegister(x) SQLITE_OK
+#endif
#ifdef _WIN32
__declspec(dllexport)
#endif
int sqlite3_fileio_init(
@@ -93,10 +954,17 @@
SQLITE_EXTENSION_INIT2(pApi);
(void)pzErrMsg; /* Unused parameter */
rc = sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0,
readfileFunc, 0, 0);
if( rc==SQLITE_OK ){
- rc = sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0,
+ rc = sqlite3_create_function(db, "writefile", -1, SQLITE_UTF8, 0,
writefileFunc, 0, 0);
}
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0,
+ lsModeFunc, 0, 0);
+ }
+ if( rc==SQLITE_OK ){
+ rc = fsdirRegister(db);
+ }
return rc;
}
Index: ext/misc/json1.c
==================================================================
--- ext/misc/json1.c
+++ ext/misc/json1.c
@@ -170,10 +170,11 @@
u32 *aUp; /* Index of parent of each node */
u8 oom; /* Set to true if out of memory */
u8 nErr; /* Number of errors seen */
u16 iDepth; /* Nesting depth */
int nJson; /* Length of the zJson string in bytes */
+ u32 iHold; /* Replace cache line with the lowest iHold value */
};
/*
** Maximum nesting depth of JSON for this implementation.
**
@@ -974,11 +975,12 @@
}
/*
** Magic number used for the JSON parse cache in sqlite3_get_auxdata()
*/
-#define JSON_CACHE_ID (-429938)
+#define JSON_CACHE_ID (-429938) /* First cache entry */
+#define JSON_CACHE_SZ 4 /* Max number of cache entries */
/*
** Obtain a complete parse of the JSON found in the first argument
** of the argv array. Use the sqlite3_get_auxdata() cache for this
** parse if it is available. If the cache is not available or if it
@@ -986,36 +988,64 @@
** and also register the new parse so that it will be available for
** future sqlite3_get_auxdata() calls.
*/
static JsonParse *jsonParseCached(
sqlite3_context *pCtx,
- sqlite3_value **argv
+ sqlite3_value **argv,
+ sqlite3_context *pErrCtx
){
const char *zJson = (const char*)sqlite3_value_text(argv[0]);
int nJson = sqlite3_value_bytes(argv[0]);
JsonParse *p;
+ JsonParse *pMatch = 0;
+ int iKey;
+ int iMinKey = 0;
+ u32 iMinHold = 0xffffffff;
+ u32 iMaxHold = 0;
if( zJson==0 ) return 0;
- p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
- if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){
- p->nErr = 0;
- return p; /* The cached entry matches, so return it */
+ for(iKey=0; iKeynJson==nJson
+ && memcmp(p->zJson,zJson,nJson)==0
+ ){
+ p->nErr = 0;
+ pMatch = p;
+ }else if( p->iHoldiHold;
+ iMinKey = iKey;
+ }
+ if( p->iHold>iMaxHold ){
+ iMaxHold = p->iHold;
+ }
+ }
+ if( pMatch ){
+ pMatch->nErr = 0;
+ pMatch->iHold = iMaxHold+1;
+ return pMatch;
}
p = sqlite3_malloc( sizeof(*p) + nJson + 1 );
if( p==0 ){
sqlite3_result_error_nomem(pCtx);
return 0;
}
memset(p, 0, sizeof(*p));
p->zJson = (char*)&p[1];
memcpy((char*)p->zJson, zJson, nJson+1);
- if( jsonParse(p, pCtx, p->zJson) ){
+ if( jsonParse(p, pErrCtx, p->zJson) ){
sqlite3_free(p);
return 0;
}
p->nJson = nJson;
- sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree);
- return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID);
+ p->iHold = iMaxHold+1;
+ sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p,
+ (void(*)(void*))jsonParseFree);
+ return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey);
}
/*
** Compare the OBJECT label at pNode against zKey,nKey. Return true on
** a match.
@@ -1384,11 +1414,11 @@
JsonParse *p; /* The parse */
sqlite3_int64 n = 0;
u32 i;
JsonNode *pNode;
- p = jsonParseCached(ctx, argv);
+ p = jsonParseCached(ctx, argv, ctx);
if( p==0 ) return;
assert( p->nNode );
if( argc==2 ){
const char *zPath = (const char*)sqlite3_value_text(argv[1]);
pNode = jsonLookup(p, zPath, 0, ctx);
@@ -1425,11 +1455,11 @@
const char *zPath;
JsonString jx;
int i;
if( argc<2 ) return;
- p = jsonParseCached(ctx, argv);
+ p = jsonParseCached(ctx, argv, ctx);
if( p==0 ) return;
jsonInit(&jx, ctx);
jsonAppendChar(&jx, '[');
for(i=1; iaNode;
}
if( pNode ){
sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC);
}
- jsonParseReset(&x);
}
/*
** json_valid(JSON)
**
@@ -1761,19 +1790,14 @@
static void jsonValidFunc(
sqlite3_context *ctx,
int argc,
sqlite3_value **argv
){
- JsonParse x; /* The parse */
- int rc = 0;
-
+ JsonParse *p; /* The parse */
UNUSED_PARAM(argc);
- if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){
- rc = 1;
- }
- jsonParseReset(&x);
- sqlite3_result_int(ctx, rc);
+ p = jsonParseCached(ctx, argv, 0);
+ sqlite3_result_int(ctx, p!=0);
}
/****************************************************************************
** Aggregate SQL function implementations
@@ -1800,29 +1824,79 @@
pStr->pCtx = ctx;
}
jsonAppendValue(pStr, argv[0]);
}
}
-static void jsonArrayFinal(sqlite3_context *ctx){
+static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){
JsonString *pStr;
pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
if( pStr ){
pStr->pCtx = ctx;
jsonAppendChar(pStr, ']');
if( pStr->bErr ){
if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
assert( pStr->bStatic );
- }else{
- sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+ }else if( isFinal ){
+ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
pStr->bStatic = 1;
+ }else{
+ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+ pStr->nUsed--;
}
}else{
sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC);
}
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
+static void jsonArrayValue(sqlite3_context *ctx){
+ jsonArrayCompute(ctx, 0);
+}
+static void jsonArrayFinal(sqlite3_context *ctx){
+ jsonArrayCompute(ctx, 1);
+}
+
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** This method works for both json_group_array() and json_group_object().
+** It works by removing the first element of the group by searching forward
+** to the first comma (",") that is not within a string and deleting all
+** text through that comma.
+*/
+static void jsonGroupInverse(
+ sqlite3_context *ctx,
+ int argc,
+ sqlite3_value **argv
+){
+ int i;
+ int inStr = 0;
+ char *z;
+ JsonString *pStr;
+ UNUSED_PARAM(argc);
+ UNUSED_PARAM(argv);
+ pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
+#ifdef NEVER
+ /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will
+ ** always have been called to initalize it */
+ if( NEVER(!pStr) ) return;
+#endif
+ z = pStr->zBuf;
+ for(i=1; z[i]!=',' || inStr; i++){
+ assert( inUsed );
+ if( z[i]=='"' ){
+ inStr = !inStr;
+ }else if( z[i]=='\\' ){
+ i++;
+ }
+ }
+ pStr->nUsed -= i;
+ memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1);
+}
+#else
+# define jsonGroupInverse 0
+#endif
+
/*
** json_group_obj(NAME,VALUE)
**
** Return a JSON object composed of all names and values in the aggregate.
@@ -1850,28 +1924,38 @@
jsonAppendString(pStr, z, n);
jsonAppendChar(pStr, ':');
jsonAppendValue(pStr, argv[1]);
}
}
-static void jsonObjectFinal(sqlite3_context *ctx){
+static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){
JsonString *pStr;
pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0);
if( pStr ){
jsonAppendChar(pStr, '}');
if( pStr->bErr ){
if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx);
assert( pStr->bStatic );
- }else{
- sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed,
+ }else if( isFinal ){
+ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed,
pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free);
pStr->bStatic = 1;
+ }else{
+ sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT);
+ pStr->nUsed--;
}
}else{
sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC);
}
sqlite3_result_subtype(ctx, JSON_SUBTYPE);
}
+static void jsonObjectValue(sqlite3_context *ctx){
+ jsonObjectCompute(ctx, 0);
+}
+static void jsonObjectFinal(sqlite3_context *ctx){
+ jsonObjectCompute(ctx, 1);
+}
+
#ifndef SQLITE_OMIT_VIRTUALTABLE
/****************************************************************************
** The json_each virtual table
@@ -1908,10 +1992,13 @@
#define JEACH_ATOM 3
#define JEACH_ID 4
#define JEACH_PARENT 5
#define JEACH_FULLKEY 6
#define JEACH_PATH 7
+/* The xBestIndex method assumes that the JSON and ROOT columns are
+** the last two columns in the table. Should this ever changes, be
+** sure to update the xBestIndex method. */
#define JEACH_JSON 8
#define JEACH_ROOT 9
UNUSED_PARAM(pzErr);
UNUSED_PARAM(argv);
@@ -2116,11 +2203,11 @@
}else{
jsonAppendChar(&x, '$');
}
if( p->eType==JSON_ARRAY ){
jsonPrintf(30, &x, "[%d]", p->iRowid);
- }else{
+ }else if( p->eType==JSON_OBJECT ){
jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1);
}
}
jsonResult(&x);
break;
@@ -2165,39 +2252,58 @@
*/
static int jsonEachBestIndex(
sqlite3_vtab *tab,
sqlite3_index_info *pIdxInfo
){
- int i;
- int jsonIdx = -1;
- int rootIdx = -1;
+ int i; /* Loop counter or computed array index */
+ int aIdx[2]; /* Index of constraints for JSON and ROOT */
+ int unusableMask = 0; /* Mask of unusable JSON and ROOT constraints */
+ int idxMask = 0; /* Mask of usable == constraints JSON and ROOT */
const struct sqlite3_index_constraint *pConstraint;
- UNUSED_PARAM(tab);
- pConstraint = pIdxInfo->aConstraint;
- for(i=0; inConstraint; i++, pConstraint++){
- if( pConstraint->usable==0 ) continue;
- if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
- switch( pConstraint->iColumn ){
- case JEACH_JSON: jsonIdx = i; break;
- case JEACH_ROOT: rootIdx = i; break;
- default: /* no-op */ break;
- }
- }
- if( jsonIdx<0 ){
- pIdxInfo->idxNum = 0;
- pIdxInfo->estimatedCost = 1e99;
- }else{
- pIdxInfo->estimatedCost = 1.0;
- pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1;
- pIdxInfo->aConstraintUsage[jsonIdx].omit = 1;
- if( rootIdx<0 ){
- pIdxInfo->idxNum = 1;
- }else{
- pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2;
- pIdxInfo->aConstraintUsage[rootIdx].omit = 1;
- pIdxInfo->idxNum = 3;
+ /* This implementation assumes that JSON and ROOT are the last two
+ ** columns in the table */
+ assert( JEACH_ROOT == JEACH_JSON+1 );
+ UNUSED_PARAM(tab);
+ aIdx[0] = aIdx[1] = -1;
+ pConstraint = pIdxInfo->aConstraint;
+ for(i=0; inConstraint; i++, pConstraint++){
+ int iCol;
+ int iMask;
+ if( pConstraint->iColumn < JEACH_JSON ) continue;
+ iCol = pConstraint->iColumn - JEACH_JSON;
+ assert( iCol==0 || iCol==1 );
+ iMask = 1 << iCol;
+ if( pConstraint->usable==0 ){
+ unusableMask |= iMask;
+ }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ aIdx[iCol] = i;
+ idxMask |= iMask;
+ }
+ }
+ if( (unusableMask & ~idxMask)!=0 ){
+ /* If there are any unusable constraints on JSON or ROOT, then reject
+ ** this entire plan */
+ return SQLITE_CONSTRAINT;
+ }
+ if( aIdx[0]<0 ){
+ /* No JSON input. Leave estimatedCost at the huge value that it was
+ ** initialized to to discourage the query planner from selecting this
+ ** plan. */
+ pIdxInfo->idxNum = 0;
+ }else{
+ pIdxInfo->estimatedCost = 1.0;
+ i = aIdx[0];
+ pIdxInfo->aConstraintUsage[i].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[i].omit = 1;
+ if( aIdx[1]<0 ){
+ pIdxInfo->idxNum = 1; /* Only JSON supplied. Plan 1 */
+ }else{
+ i = aIdx[1];
+ pIdxInfo->aConstraintUsage[i].argvIndex = 2;
+ pIdxInfo->aConstraintUsage[i].omit = 1;
+ pIdxInfo->idxNum = 3; /* Both JSON and ROOT are supplied. Plan 3 */
}
}
return SQLITE_OK;
}
@@ -2302,11 +2408,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
/* The methods of the json_tree virtual table. */
static sqlite3_module jsonTreeModule = {
0, /* iVersion */
@@ -2329,11 +2436,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
#endif /* SQLITE_OMIT_VIRTUALTABLE */
/****************************************************************************
** The following routines are the only publically visible identifiers in this
@@ -2375,13 +2483,16 @@
static const struct {
const char *zName;
int nArg;
void (*xStep)(sqlite3_context*,int,sqlite3_value**);
void (*xFinal)(sqlite3_context*);
+ void (*xValue)(sqlite3_context*);
} aAgg[] = {
- { "json_group_array", 1, jsonArrayStep, jsonArrayFinal },
- { "json_group_object", 2, jsonObjectStep, jsonObjectFinal },
+ { "json_group_array", 1,
+ jsonArrayStep, jsonArrayFinal, jsonArrayValue },
+ { "json_group_object", 2,
+ jsonObjectStep, jsonObjectFinal, jsonObjectValue },
};
#ifndef SQLITE_OMIT_VIRTUALTABLE
static const struct {
const char *zName;
sqlite3_module *pModule;
@@ -2394,15 +2505,18 @@
rc = sqlite3_create_function(db, aFunc[i].zName, aFunc[i].nArg,
SQLITE_UTF8 | SQLITE_DETERMINISTIC,
(void*)&aFunc[i].flag,
aFunc[i].xFunc, 0, 0);
}
+#ifndef SQLITE_OMIT_WINDOWFUNC
for(i=0; i
+#include
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/* memstat_vtab is a subclass of sqlite3_vtab which will
+** serve as the underlying representation of a memstat virtual table
+*/
+typedef struct memstat_vtab memstat_vtab;
+struct memstat_vtab {
+ sqlite3_vtab base; /* Base class - must be first */
+ sqlite3 *db; /* Database connection for this memstat vtab */
+};
+
+/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct memstat_cursor memstat_cursor;
+struct memstat_cursor {
+ sqlite3_vtab_cursor base; /* Base class - must be first */
+ sqlite3 *db; /* Database connection for this cursor */
+ int iRowid; /* Current row in aMemstatColumn[] */
+ int iDb; /* Which schema we are looking at */
+ int nDb; /* Number of schemas */
+ char **azDb; /* Names of all schemas */
+ sqlite3_int64 aVal[2]; /* Result values */
+};
+
+/*
+** The memstatConnect() method is invoked to create a new
+** memstat_vtab that describes the memstat virtual table.
+**
+** Think of this routine as the constructor for memstat_vtab objects.
+**
+** All this routine needs to do is:
+**
+** (1) Allocate the memstat_vtab object and initialize all fields.
+**
+** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+** result set of queries against memstat will look like.
+*/
+static int memstatConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ memstat_vtab *pNew;
+ int rc;
+
+/* Column numbers */
+#define MSV_COLUMN_NAME 0 /* Name of quantity being measured */
+#define MSV_COLUMN_SCHEMA 1 /* schema name */
+#define MSV_COLUMN_VALUE 2 /* Current value */
+#define MSV_COLUMN_HIWTR 3 /* Highwater mark */
+
+ rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)");
+ if( rc==SQLITE_OK ){
+ pNew = sqlite3_malloc( sizeof(*pNew) );
+ *ppVtab = (sqlite3_vtab*)pNew;
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(*pNew));
+ pNew->db = db;
+ }
+ return rc;
+}
+
+/*
+** This method is the destructor for memstat_cursor objects.
+*/
+static int memstatDisconnect(sqlite3_vtab *pVtab){
+ sqlite3_free(pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new memstat_cursor object.
+*/
+static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+ memstat_cursor *pCur;
+ pCur = sqlite3_malloc( sizeof(*pCur) );
+ if( pCur==0 ) return SQLITE_NOMEM;
+ memset(pCur, 0, sizeof(*pCur));
+ pCur->db = ((memstat_vtab*)p)->db;
+ *ppCursor = &pCur->base;
+ return SQLITE_OK;
+}
+
+/*
+** Clear all the schema names from a cursor
+*/
+static void memstatClearSchema(memstat_cursor *pCur){
+ int i;
+ if( pCur->azDb==0 ) return;
+ for(i=0; inDb; i++){
+ sqlite3_free(pCur->azDb[i]);
+ }
+ sqlite3_free(pCur->azDb);
+ pCur->azDb = 0;
+ pCur->nDb = 0;
+}
+
+/*
+** Fill in the azDb[] array for the cursor.
+*/
+static int memstatFindSchemas(memstat_cursor *pCur){
+ sqlite3_stmt *pStmt = 0;
+ int rc;
+ if( pCur->nDb ) return SQLITE_OK;
+ rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0);
+ if( rc ){
+ sqlite3_finalize(pStmt);
+ return rc;
+ }
+ while( sqlite3_step(pStmt)==SQLITE_ROW ){
+ char **az, *z;
+ az = sqlite3_realloc(pCur->azDb, sizeof(char*)*(pCur->nDb+1));
+ if( az==0 ){
+ memstatClearSchema(pCur);
+ return SQLITE_NOMEM;
+ }
+ pCur->azDb = az;
+ z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1));
+ if( z==0 ){
+ memstatClearSchema(pCur);
+ return SQLITE_NOMEM;
+ }
+ pCur->azDb[pCur->nDb] = z;
+ pCur->nDb++;
+ }
+ sqlite3_finalize(pStmt);
+ return SQLITE_OK;
+}
+
+
+/*
+** Destructor for a memstat_cursor.
+*/
+static int memstatClose(sqlite3_vtab_cursor *cur){
+ memstat_cursor *pCur = (memstat_cursor*)cur;
+ memstatClearSchema(pCur);
+ sqlite3_free(cur);
+ return SQLITE_OK;
+}
+
+
+/*
+** Allowed values for aMemstatColumn[].eType
+*/
+#define MSV_GSTAT 0 /* sqlite3_status64() information */
+#define MSV_DB 1 /* sqlite3_db_status() information */
+#define MSV_ZIPVFS 2 /* ZIPVFS file-control with 64-bit return */
+
+/*
+** An array of quantities that can be measured and reported by
+** this virtual table
+*/
+static const struct MemstatColumns {
+ const char *zName; /* Symbolic name */
+ unsigned char eType; /* Type of interface */
+ unsigned char mNull; /* Bitmask of which columns are NULL */
+ /* 2: dbname, 4: current, 8: hiwtr */
+ int eOp; /* Opcode */
+} aMemstatColumn[] = {
+ {"MEMORY_USED", MSV_GSTAT, 2, SQLITE_STATUS_MEMORY_USED },
+ {"MALLOC_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_MALLOC_SIZE },
+ {"MALLOC_COUNT", MSV_GSTAT, 2, SQLITE_STATUS_MALLOC_COUNT },
+ {"PAGECACHE_USED", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_USED },
+ {"PAGECACHE_OVERFLOW", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_OVERFLOW },
+ {"PAGECACHE_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_PAGECACHE_SIZE },
+ {"PARSER_STACK", MSV_GSTAT, 6, SQLITE_STATUS_PARSER_STACK },
+ {"DB_LOOKASIDE_USED", MSV_DB, 2, SQLITE_DBSTATUS_LOOKASIDE_USED },
+ {"DB_LOOKASIDE_HIT", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_HIT },
+ {"DB_LOOKASIDE_MISS_SIZE", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE},
+ {"DB_LOOKASIDE_MISS_FULL", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL},
+ {"DB_CACHE_USED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED },
+#if SQLITE_VERSION_NUMBER >= 3140000
+ {"DB_CACHE_USED_SHARED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED_SHARED },
+#endif
+ {"DB_SCHEMA_USED", MSV_DB, 10, SQLITE_DBSTATUS_SCHEMA_USED },
+ {"DB_STMT_USED", MSV_DB, 10, SQLITE_DBSTATUS_STMT_USED },
+ {"DB_CACHE_HIT", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_HIT },
+ {"DB_CACHE_MISS", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_MISS },
+ {"DB_CACHE_WRITE", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_WRITE },
+#if SQLITE_VERSION_NUMBER >= 3230000
+ {"DB_CACHE_SPILL", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_SPILL },
+#endif
+ {"DB_DEFERRED_FKS", MSV_DB, 10, SQLITE_DBSTATUS_DEFERRED_FKS },
+#ifdef SQLITE_ENABLE_ZIPVFS
+ {"ZIPVFS_CACHE_USED", MSV_ZIPVFS, 8, 231454 },
+ {"ZIPVFS_CACHE_HIT", MSV_ZIPVFS, 8, 231455 },
+ {"ZIPVFS_CACHE_MISS", MSV_ZIPVFS, 8, 231456 },
+ {"ZIPVFS_CACHE_WRITE", MSV_ZIPVFS, 8, 231457 },
+ {"ZIPVFS_DIRECT_READ", MSV_ZIPVFS, 8, 231458 },
+ {"ZIPVFS_DIRECT_BYTES", MSV_ZIPVFS, 8, 231459 },
+#endif /* SQLITE_ENABLE_ZIPVFS */
+};
+#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0]))
+
+/*
+** Advance a memstat_cursor to its next row of output.
+*/
+static int memstatNext(sqlite3_vtab_cursor *cur){
+ memstat_cursor *pCur = (memstat_cursor*)cur;
+ int i;
+ assert( pCur->iRowid<=MSV_NROW );
+ while(1){
+ i = (int)pCur->iRowid - 1;
+ if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){
+ pCur->iRowid++;
+ if( pCur->iRowid>MSV_NROW ) return SQLITE_OK; /* End of the table */
+ pCur->iDb = 0;
+ i++;
+ }
+ pCur->aVal[0] = 0;
+ pCur->aVal[1] = 0;
+ switch( aMemstatColumn[i].eType ){
+ case MSV_GSTAT: {
+ if( sqlite3_libversion_number()>=3010000 ){
+ sqlite3_status64(aMemstatColumn[i].eOp,
+ &pCur->aVal[0], &pCur->aVal[1],0);
+ }else{
+ int xCur, xHiwtr;
+ sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+ pCur->aVal[0] = xCur;
+ pCur->aVal[1] = xHiwtr;
+ }
+ break;
+ }
+ case MSV_DB: {
+ int xCur, xHiwtr;
+ sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0);
+ pCur->aVal[0] = xCur;
+ pCur->aVal[1] = xHiwtr;
+ break;
+ }
+ case MSV_ZIPVFS: {
+ int rc;
+ rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb],
+ aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]);
+ if( rc!=SQLITE_OK ) continue;
+ break;
+ }
+ }
+ break;
+ }
+ return SQLITE_OK;
+}
+
+
+/*
+** Return values of columns for the row at which the memstat_cursor
+** is currently pointing.
+*/
+static int memstatColumn(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int iCol /* Which column to return */
+){
+ memstat_cursor *pCur = (memstat_cursor*)cur;
+ int i;
+ assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW );
+ i = (int)pCur->iRowid - 1;
+ if( (aMemstatColumn[i].mNull & (1<azDb[pCur->iDb], -1, 0);
+ break;
+ }
+ case MSV_COLUMN_VALUE: {
+ sqlite3_result_int64(ctx, pCur->aVal[0]);
+ break;
+ }
+ case MSV_COLUMN_HIWTR: {
+ sqlite3_result_int64(ctx, pCur->aVal[1]);
+ break;
+ }
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** rowid is the same as the output value.
+*/
+static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+ memstat_cursor *pCur = (memstat_cursor*)cur;
+ *pRowid = pCur->iRowid*1000 + pCur->iDb;
+ return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int memstatEof(sqlite3_vtab_cursor *cur){
+ memstat_cursor *pCur = (memstat_cursor*)cur;
+ return pCur->iRowid>MSV_NROW;
+}
+
+/*
+** This method is called to "rewind" the memstat_cursor object back
+** to the first row of output. This method is always called at least
+** once prior to any call to memstatColumn() or memstatRowid() or
+** memstatEof().
+*/
+static int memstatFilter(
+ sqlite3_vtab_cursor *pVtabCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ memstat_cursor *pCur = (memstat_cursor *)pVtabCursor;
+ int rc = memstatFindSchemas(pCur);
+ if( rc ) return rc;
+ pCur->iRowid = 0;
+ pCur->iDb = 0;
+ return memstatNext(pVtabCursor);
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the memstat virtual table. This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int memstatBestIndex(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ pIdxInfo->estimatedCost = (double)500;
+ pIdxInfo->estimatedRows = 500;
+ return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the
+** memstat virtual table.
+*/
+static sqlite3_module memstatModule = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ memstatConnect, /* xConnect */
+ memstatBestIndex, /* xBestIndex */
+ memstatDisconnect, /* xDisconnect */
+ 0, /* xDestroy */
+ memstatOpen, /* xOpen - open a cursor */
+ memstatClose, /* xClose - close a cursor */
+ memstatFilter, /* xFilter - configure scan constraints */
+ memstatNext, /* xNext - advance a cursor */
+ memstatEof, /* xEof - check for end of scan */
+ memstatColumn, /* xColumn - read data */
+ memstatRowid, /* xRowid - read data */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindMethod */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ 0, /* xShadowName */
+};
+
+#endif /* SQLITE_OMIT_VIRTUALTABLE */
+
+int sqlite3MemstatVtabInit(sqlite3 *db){
+ int rc = SQLITE_OK;
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0);
+#endif
+ return rc;
+}
+
+#ifndef SQLITE_CORE
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_memstat_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ int rc = SQLITE_OK;
+ SQLITE_EXTENSION_INIT2(pApi);
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+ rc = sqlite3MemstatVtabInit(db);
+#endif
+ return rc;
+}
+#endif /* SQLITE_CORE */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */
Index: ext/misc/memvfs.c
==================================================================
--- ext/misc/memvfs.c
+++ ext/misc/memvfs.c
@@ -8,27 +8,37 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
-** This is an in-memory read-only VFS implementation. The application
-** supplies a block of memory which is the database file, and this VFS
-** uses that block of memory.
+** This is an in-memory VFS implementation. The application supplies
+** a chunk of memory to hold the database file.
**
-** Because there is no place to store journals and no good way to lock
-** the "file", this VFS is read-only.
+** Because there is place to store a rollback or wal journal, the database
+** must use one of journal_mode=MEMORY or journal_mode=NONE.
**
** USAGE:
**
-** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336", &db,
-** SQLITE_OPEN_READONLY | SQLITE_OPEN_URI,
+** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db,
+** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI,
** "memvfs");
**
-** The ptr= and sz= query parameters are required or the open will fail.
-** The ptr= parameter gives the memory address of the buffer holding the
-** read-only database and sz= gives the size of the database. The parameter
-** values may be in hexadecimal or decimal. The filename is ignored.
+** These are the query parameters:
+**
+** ptr= The address of the memory buffer that holds the database.
+**
+** sz= The current size the database file
+**
+** maxsz= The maximum size of the database. In other words, the
+** amount of space allocated for the ptr= buffer.
+**
+** freeonclose= If true, then sqlite3_free() is called on the ptr=
+** value when the connection closes.
+**
+** The ptr= and sz= query parameters are required. If maxsz= is omitted,
+** then it defaults to the sz= value. Parameter values can be in either
+** decimal or hexadecimal. The filename in the URI is ignored.
*/
#include
SQLITE_EXTENSION_INIT1
#include
#include
@@ -47,11 +57,13 @@
/* An open file */
struct MemFile {
sqlite3_file base; /* IO methods */
sqlite3_int64 sz; /* Size of the file */
+ sqlite3_int64 szMax; /* Space allocated to aData */
unsigned char *aData; /* content of the file */
+ int bFreeOnClose; /* Invoke sqlite3_free() on aData at close */
};
/*
** Methods for MemFile
*/
@@ -142,10 +154,12 @@
**
** The pData pointer is owned by the application, so there is nothing
** to free.
*/
static int memClose(sqlite3_file *pFile){
+ MemFile *p = (MemFile *)pFile;
+ if( p->bFreeOnClose ) sqlite3_free(p->aData);
return SQLITE_OK;
}
/*
** Read data from an mem-file.
@@ -168,25 +182,38 @@
sqlite3_file *pFile,
const void *z,
int iAmt,
sqlite_int64 iOfst
){
- return SQLITE_READONLY;
+ MemFile *p = (MemFile *)pFile;
+ if( iOfst+iAmt>p->sz ){
+ if( iOfst+iAmt>p->szMax ) return SQLITE_FULL;
+ if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz);
+ p->sz = iOfst+iAmt;
+ }
+ memcpy(p->aData+iOfst, z, iAmt);
+ return SQLITE_OK;
}
/*
** Truncate an mem-file.
*/
static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){
- return SQLITE_READONLY;
+ MemFile *p = (MemFile *)pFile;
+ if( size>p->sz ){
+ if( size>p->szMax ) return SQLITE_FULL;
+ memset(p->aData+p->sz, 0, size-p->sz);
+ }
+ p->sz = size;
+ return SQLITE_OK;
}
/*
** Sync an mem-file.
*/
static int memSync(sqlite3_file *pFile, int flags){
- return SQLITE_READONLY;
+ return SQLITE_OK;
}
/*
** Return the current file-size of an mem-file.
*/
@@ -198,11 +225,11 @@
/*
** Lock an mem-file.
*/
static int memLock(sqlite3_file *pFile, int eLock){
- return SQLITE_READONLY;
+ return SQLITE_OK;
}
/*
** Unlock an mem-file.
*/
@@ -240,11 +267,14 @@
/*
** Return the device characteristic flags supported by an mem-file.
*/
static int memDeviceCharacteristics(sqlite3_file *pFile){
- return SQLITE_IOCAP_IMMUTABLE;
+ return SQLITE_IOCAP_ATOMIC |
+ SQLITE_IOCAP_POWERSAFE_OVERWRITE |
+ SQLITE_IOCAP_SAFE_APPEND |
+ SQLITE_IOCAP_SEQUENTIAL;
}
/* Create a shared memory file mapping */
static int memShmMap(
sqlite3_file *pFile,
@@ -251,16 +281,16 @@
int iPg,
int pgsz,
int bExtend,
void volatile **pp
){
- return SQLITE_READONLY;
+ return SQLITE_IOERR_SHMMAP;
}
/* Perform locking on a shared-memory segment */
static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){
- return SQLITE_READONLY;
+ return SQLITE_IOERR_SHMLOCK;
}
/* Memory barrier operation on shared memory */
static void memShmBarrier(sqlite3_file *pFile){
return;
@@ -303,10 +333,13 @@
if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN;
p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0);
if( p->aData==0 ) return SQLITE_CANTOPEN;
p->sz = sqlite3_uri_int64(zName,"sz",0);
if( p->sz<0 ) return SQLITE_CANTOPEN;
+ p->szMax = sqlite3_uri_int64(zName,"max",p->sz);
+ if( p->szMaxsz ) return SQLITE_CANTOPEN;
+ p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0);
pFile->pMethods = &mem_io_methods;
return SQLITE_OK;
}
/*
@@ -313,11 +346,11 @@
** Delete the file located at zPath. If the dirSync argument is true,
** ensure the file-system modifications are synced to disk before
** returning.
*/
static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){
- return SQLITE_READONLY;
+ return SQLITE_IOERR_DELETE;
}
/*
** Test for access permissions. Return true if the requested permission
** is available, or false otherwise.
@@ -326,18 +359,11 @@
sqlite3_vfs *pVfs,
const char *zPath,
int flags,
int *pResOut
){
- /* The spec says there are three possible values for flags. But only
- ** two of them are actually used */
- assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE );
- if( flags==SQLITE_ACCESS_READWRITE ){
- *pResOut = 0;
- }else{
- *pResOut = 1;
- }
+ *pResOut = 0;
return SQLITE_OK;
}
/*
** Populate buffer zOut with the full canonical pathname corresponding
@@ -414,54 +440,108 @@
return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p);
}
#ifdef MEMVFS_TEST
/*
-** memload(FILENAME)
+** memvfs_from_file(FILENAME, MAXSIZE)
**
** This an SQL function used to help in testing the memvfs VFS. The
** function reads the content of a file into memory and then returns
-** a string that gives the locate and size of the in-memory buffer.
+** a URI that can be handed to ATTACH to attach the memory buffer as
+** a database. Example:
+**
+** ATTACH memvfs_from_file('test.db',1048576) AS inmem;
+**
+** The optional MAXSIZE argument gives the size of the memory allocation
+** used to hold the database. If omitted, it defaults to the size of the
+** file on disk.
*/
#include
-static void memvfsMemloadFunc(
+static void memvfsFromFileFunc(
sqlite3_context *context,
int argc,
sqlite3_value **argv
){
unsigned char *p;
sqlite3_int64 sz;
+ sqlite3_int64 szMax;
FILE *in;
const char *zFilename = (const char*)sqlite3_value_text(argv[0]);
- char zReturn[100];
+ char *zUri;
if( zFilename==0 ) return;
in = fopen(zFilename, "rb");
if( in==0 ) return;
fseek(in, 0, SEEK_END);
- sz = ftell(in);
+ szMax = sz = ftell(in);
rewind(in);
- p = sqlite3_malloc( sz );
+ if( argc>=2 ){
+ szMax = sqlite3_value_int64(argv[1]);
+ if( szMaxzName,"memvfs")!=0 ) return;
+ rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p);
+ if( rc ) return;
+ fwrite(p->aData, 1, (size_t)p->sz, out);
+ fclose(out);
}
+#endif /* MEMVFS_TEST */
+
+#ifdef MEMVFS_TEST
/* Called for each new database connection */
static int memvfsRegister(
sqlite3 *db,
- const char **pzErrMsg,
+ char **pzErrMsg,
const struct sqlite3_api_routines *pThunk
){
- return sqlite3_create_function(db, "memload", 1, SQLITE_UTF8, 0,
- memvfsMemloadFunc, 0, 0);
+ sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0,
+ memvfsFromFileFunc, 0, 0);
+ sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0,
+ memvfsFromFileFunc, 0, 0);
+ sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0,
+ memvfsToFileFunc, 0, 0);
+ return SQLITE_OK;
}
#endif /* MEMVFS_TEST */
#ifdef _WIN32
@@ -483,9 +563,12 @@
rc = sqlite3_vfs_register(&mem_vfs, 1);
#ifdef MEMVFS_TEST
if( rc==SQLITE_OK ){
rc = sqlite3_auto_extension((void(*)(void))memvfsRegister);
}
+ if( rc==SQLITE_OK ){
+ rc = memvfsRegister(db, pzErrMsg, pApi);
+ }
#endif
if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY;
return rc;
}
ADDED ext/misc/normalize.c
Index: ext/misc/normalize.c
==================================================================
--- /dev/null
+++ ext/misc/normalize.c
@@ -0,0 +1,707 @@
+/*
+** 2018-01-08
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains code to implement the sqlite3_normalize() function.
+**
+** char *sqlite3_normalize(const char *zSql);
+**
+** This function takes an SQL string as input and returns a "normalized"
+** version of that string in memory obtained from sqlite3_malloc64(). The
+** caller is responsible for ensuring that the returned memory is freed.
+**
+** If a memory allocation error occurs, this routine returns NULL.
+**
+** The normalization consists of the following transformations:
+**
+** (1) Convert every literal (string, blob literal, numeric constant,
+** or "NULL" constant) into a ?
+**
+** (2) Remove all superfluous whitespace, including comments. Change
+** all required whitespace to a single space character.
+**
+** (3) Lowercase all ASCII characters.
+**
+** (4) If an IN or NOT IN operator is followed by a list of 1 or more
+** values, convert that list into "(?,?,?)".
+**
+** The purpose of normalization is two-fold:
+**
+** (1) Sanitize queries by removing potentially private or sensitive
+** information contained in literals.
+**
+** (2) Identify structurally identical queries by comparing their
+** normalized forms.
+**
+** Command-Line Utility
+** --------------------
+**
+** This file also contains code for a command-line utility that converts
+** SQL queries in text files into their normalized forms. To build the
+** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI
+** and link it against the SQLite library.
+*/
+#include
+#include
+
+/*
+** Implementation note:
+**
+** Much of the tokenizer logic is copied out of the tokenize.c source file
+** of SQLite. That logic could be simplified for this particular application,
+** but that would impose a risk of introducing subtle errors. It is best to
+** keep the code as close to the original as possible.
+**
+** The tokenize code is in sync with the SQLite core as of 2018-01-08.
+** Any future changes to the core tokenizer might require corresponding
+** adjustments to the tokenizer logic in this module.
+*/
+
+
+/* Character classes for tokenizing
+**
+** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented
+** using a lookup table, whereas a switch() directly on c uses a binary search.
+** The lookup table is much faster. To maximize speed, and to ensure that
+** a lookup table is used, all of the classes need to be small integers and
+** all of them need to be used within the switch.
+*/
+#define CC_X 0 /* The letter 'x', or start of BLOB literal */
+#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */
+#define CC_ID 2 /* unicode characters usable in IDs */
+#define CC_DIGIT 3 /* Digits */
+#define CC_DOLLAR 4 /* '$' */
+#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */
+#define CC_VARNUM 6 /* '?'. Numeric SQL variables */
+#define CC_SPACE 7 /* Space characters */
+#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */
+#define CC_QUOTE2 9 /* '['. [...] style quoted ids */
+#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */
+#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */
+#define CC_LT 12 /* '<'. Part of < or <= or <> */
+#define CC_GT 13 /* '>'. Part of > or >= */
+#define CC_EQ 14 /* '='. Part of = or == */
+#define CC_BANG 15 /* '!'. Part of != */
+#define CC_SLASH 16 /* '/'. / or c-style comment */
+#define CC_LP 17 /* '(' */
+#define CC_RP 18 /* ')' */
+#define CC_SEMI 19 /* ';' */
+#define CC_PLUS 20 /* '+' */
+#define CC_STAR 21 /* '*' */
+#define CC_PERCENT 22 /* '%' */
+#define CC_COMMA 23 /* ',' */
+#define CC_AND 24 /* '&' */
+#define CC_TILDA 25 /* '~' */
+#define CC_DOT 26 /* '.' */
+#define CC_ILLEGAL 27 /* Illegal character */
+
+static const unsigned char aiClass[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */
+/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27,
+/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
+/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16,
+/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6,
+/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1,
+/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27,
+/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2
+};
+
+/* An array to map all upper-case characters into their corresponding
+** lower-case character.
+**
+** SQLite only considers US-ASCII (or EBCDIC) characters. We do not
+** handle case conversions for the UTF character set since the tables
+** involved are nearly as big or bigger than SQLite itself.
+*/
+static const unsigned char sqlite3UpperToLower[] = {
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+ 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+ 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
+ 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
+ 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
+ 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
+ 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
+ 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
+ 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
+ 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
+ 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
+ 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
+ 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
+ 252,253,254,255
+};
+
+/*
+** The following 256 byte lookup table is used to support SQLites built-in
+** equivalents to the following standard library functions:
+**
+** isspace() 0x01
+** isalpha() 0x02
+** isdigit() 0x04
+** isalnum() 0x06
+** isxdigit() 0x08
+** toupper() 0x20
+** SQLite identifier character 0x40
+** Quote character 0x80
+**
+** Bit 0x20 is set if the mapped character requires translation to upper
+** case. i.e. if the character is a lower-case ASCII character.
+** If x is a lower-case ASCII character, then its upper-case equivalent
+** is (x - 0x20). Therefore toupper() can be implemented as:
+**
+** (x & ~(map[x]&0x20))
+**
+** The equivalent of tolower() is implemented using the sqlite3UpperToLower[]
+** array. tolower() is used more often than toupper() by SQLite.
+**
+** Bit 0x40 is set if the character is non-alphanumeric and can be used in an
+** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any
+** non-ASCII UTF character. Hence the test for whether or not a character is
+** part of an identifier is 0x46.
+*/
+static const unsigned char sqlite3CtypeMap[256] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */
+ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */
+ 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */
+ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */
+ 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */
+
+ 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */
+ 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */
+ 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */
+ 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */
+ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */
+ 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */
+ 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */
+
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */
+
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */
+ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */
+};
+#define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20))
+#define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01)
+#define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06)
+#define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02)
+#define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04)
+#define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
+#define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)])
+#define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+
+
+/*
+** If X is a character that can be used in an identifier then
+** IdChar(X) will be true. Otherwise it is false.
+**
+** For ASCII, any character with the high-order bit set is
+** allowed in an identifier. For 7-bit characters,
+** sqlite3IsIdChar[X] must be 1.
+**
+** For EBCDIC, the rules are more complex but have the same
+** end result.
+**
+** Ticket #1066. the SQL standard does not allow '$' in the
+** middle of identifiers. But many SQL implementations do.
+** SQLite will allow '$' in identifiers for compatibility.
+** But the feature is undocumented.
+*/
+#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0)
+
+/*
+** Ignore testcase() macros
+*/
+#define testcase(X)
+
+/*
+** Token values
+*/
+#define TK_SPACE 0
+#define TK_NAME 1
+#define TK_LITERAL 2
+#define TK_PUNCT 3
+#define TK_ERROR 4
+
+#define TK_MINUS TK_PUNCT
+#define TK_LP TK_PUNCT
+#define TK_RP TK_PUNCT
+#define TK_SEMI TK_PUNCT
+#define TK_PLUS TK_PUNCT
+#define TK_STAR TK_PUNCT
+#define TK_SLASH TK_PUNCT
+#define TK_REM TK_PUNCT
+#define TK_EQ TK_PUNCT
+#define TK_LE TK_PUNCT
+#define TK_NE TK_PUNCT
+#define TK_LSHIFT TK_PUNCT
+#define TK_LT TK_PUNCT
+#define TK_GE TK_PUNCT
+#define TK_RSHIFT TK_PUNCT
+#define TK_GT TK_PUNCT
+#define TK_GE TK_PUNCT
+#define TK_BITOR TK_PUNCT
+#define TK_CONCAT TK_PUNCT
+#define TK_COMMA TK_PUNCT
+#define TK_BITAND TK_PUNCT
+#define TK_BITNOT TK_PUNCT
+#define TK_STRING TK_LITERAL
+#define TK_ID TK_NAME
+#define TK_ILLEGAL TK_ERROR
+#define TK_DOT TK_PUNCT
+#define TK_INTEGER TK_LITERAL
+#define TK_FLOAT TK_LITERAL
+#define TK_VARIABLE TK_LITERAL
+#define TK_BLOB TK_LITERAL
+
+/*
+** Return the length (in bytes) of the token that begins at z[0].
+** Store the token type in *tokenType before returning.
+*/
+static int sqlite3GetToken(const unsigned char *z, int *tokenType){
+ int i, c;
+ switch( aiClass[*z] ){ /* Switch on the character-class of the first byte
+ ** of the token. See the comment on the CC_ defines
+ ** above. */
+ case CC_SPACE: {
+ for(i=1; sqlite3Isspace(z[i]); i++){}
+ *tokenType = TK_SPACE;
+ return i;
+ }
+ case CC_MINUS: {
+ if( z[1]=='-' ){
+ for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
+ *tokenType = TK_SPACE;
+ return i;
+ }
+ *tokenType = TK_MINUS;
+ return 1;
+ }
+ case CC_LP: {
+ *tokenType = TK_LP;
+ return 1;
+ }
+ case CC_RP: {
+ *tokenType = TK_RP;
+ return 1;
+ }
+ case CC_SEMI: {
+ *tokenType = TK_SEMI;
+ return 1;
+ }
+ case CC_PLUS: {
+ *tokenType = TK_PLUS;
+ return 1;
+ }
+ case CC_STAR: {
+ *tokenType = TK_STAR;
+ return 1;
+ }
+ case CC_SLASH: {
+ if( z[1]!='*' || z[2]==0 ){
+ *tokenType = TK_SLASH;
+ return 1;
+ }
+ for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
+ if( c ) i++;
+ *tokenType = TK_SPACE;
+ return i;
+ }
+ case CC_PERCENT: {
+ *tokenType = TK_REM;
+ return 1;
+ }
+ case CC_EQ: {
+ *tokenType = TK_EQ;
+ return 1 + (z[1]=='=');
+ }
+ case CC_LT: {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_LE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_NE;
+ return 2;
+ }else if( c=='<' ){
+ *tokenType = TK_LSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_LT;
+ return 1;
+ }
+ }
+ case CC_GT: {
+ if( (c=z[1])=='=' ){
+ *tokenType = TK_GE;
+ return 2;
+ }else if( c=='>' ){
+ *tokenType = TK_RSHIFT;
+ return 2;
+ }else{
+ *tokenType = TK_GT;
+ return 1;
+ }
+ }
+ case CC_BANG: {
+ if( z[1]!='=' ){
+ *tokenType = TK_ILLEGAL;
+ return 1;
+ }else{
+ *tokenType = TK_NE;
+ return 2;
+ }
+ }
+ case CC_PIPE: {
+ if( z[1]!='|' ){
+ *tokenType = TK_BITOR;
+ return 1;
+ }else{
+ *tokenType = TK_CONCAT;
+ return 2;
+ }
+ }
+ case CC_COMMA: {
+ *tokenType = TK_COMMA;
+ return 1;
+ }
+ case CC_AND: {
+ *tokenType = TK_BITAND;
+ return 1;
+ }
+ case CC_TILDA: {
+ *tokenType = TK_BITNOT;
+ return 1;
+ }
+ case CC_QUOTE: {
+ int delim = z[0];
+ testcase( delim=='`' );
+ testcase( delim=='\'' );
+ testcase( delim=='"' );
+ for(i=1; (c=z[i])!=0; i++){
+ if( c==delim ){
+ if( z[i+1]==delim ){
+ i++;
+ }else{
+ break;
+ }
+ }
+ }
+ if( c=='\'' ){
+ *tokenType = TK_STRING;
+ return i+1;
+ }else if( c!=0 ){
+ *tokenType = TK_ID;
+ return i+1;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ return i;
+ }
+ }
+ case CC_DOT: {
+ if( !sqlite3Isdigit(z[1]) ){
+ *tokenType = TK_DOT;
+ return 1;
+ }
+ /* If the next character is a digit, this is a floating point
+ ** number that begins with ".". Fall thru into the next case */
+ }
+ case CC_DIGIT: {
+ *tokenType = TK_INTEGER;
+ if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){
+ for(i=3; sqlite3Isxdigit(z[i]); i++){}
+ return i;
+ }
+ for(i=0; sqlite3Isdigit(z[i]); i++){}
+ if( z[i]=='.' ){
+ i++;
+ while( sqlite3Isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+ if( (z[i]=='e' || z[i]=='E') &&
+ ( sqlite3Isdigit(z[i+1])
+ || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2]))
+ )
+ ){
+ i += 2;
+ while( sqlite3Isdigit(z[i]) ){ i++; }
+ *tokenType = TK_FLOAT;
+ }
+ while( IdChar(z[i]) ){
+ *tokenType = TK_ILLEGAL;
+ i++;
+ }
+ return i;
+ }
+ case CC_QUOTE2: {
+ for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
+ *tokenType = c==']' ? TK_ID : TK_ILLEGAL;
+ return i;
+ }
+ case CC_VARNUM: {
+ *tokenType = TK_VARIABLE;
+ for(i=1; sqlite3Isdigit(z[i]); i++){}
+ return i;
+ }
+ case CC_DOLLAR:
+ case CC_VARALPHA: {
+ int n = 0;
+ testcase( z[0]=='$' ); testcase( z[0]=='@' );
+ testcase( z[0]==':' ); testcase( z[0]=='#' );
+ *tokenType = TK_VARIABLE;
+ for(i=1; (c=z[i])!=0; i++){
+ if( IdChar(c) ){
+ n++;
+ }else if( c=='(' && n>0 ){
+ do{
+ i++;
+ }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' );
+ if( c==')' ){
+ i++;
+ }else{
+ *tokenType = TK_ILLEGAL;
+ }
+ break;
+ }else if( c==':' && z[i+1]==':' ){
+ i++;
+ }else{
+ break;
+ }
+ }
+ if( n==0 ) *tokenType = TK_ILLEGAL;
+ return i;
+ }
+ case CC_KYWD: {
+ for(i=1; aiClass[z[i]]<=CC_KYWD; i++){}
+ if( IdChar(z[i]) ){
+ /* This token started out using characters that can appear in keywords,
+ ** but z[i] is a character not allowed within keywords, so this must
+ ** be an identifier instead */
+ i++;
+ break;
+ }
+ *tokenType = TK_ID;
+ return i;
+ }
+ case CC_X: {
+ testcase( z[0]=='x' ); testcase( z[0]=='X' );
+ if( z[1]=='\'' ){
+ *tokenType = TK_BLOB;
+ for(i=2; sqlite3Isxdigit(z[i]); i++){}
+ if( z[i]!='\'' || i%2 ){
+ *tokenType = TK_ILLEGAL;
+ while( z[i] && z[i]!='\'' ){ i++; }
+ }
+ if( z[i] ) i++;
+ return i;
+ }
+ /* If it is not a BLOB literal, then it must be an ID, since no
+ ** SQL keywords start with the letter 'x'. Fall through */
+ }
+ case CC_ID: {
+ i = 1;
+ break;
+ }
+ default: {
+ *tokenType = TK_ILLEGAL;
+ return 1;
+ }
+ }
+ while( IdChar(z[i]) ){ i++; }
+ *tokenType = TK_ID;
+ return i;
+}
+
+char *sqlite3_normalize(const char *zSql){
+ char *z; /* The output string */
+ sqlite3_int64 nZ; /* Size of the output string in bytes */
+ sqlite3_int64 nSql; /* Size of the input string in bytes */
+ int i; /* Next character to read from zSql[] */
+ int j; /* Next slot to fill in on z[] */
+ int tokenType; /* Type of the next token */
+ int n; /* Size of the next token */
+ int k; /* Loop counter */
+
+ nSql = strlen(zSql);
+ nZ = nSql;
+ z = sqlite3_malloc64( nZ+2 );
+ if( z==0 ) return 0;
+ for(i=j=0; zSql[i]; i += n){
+ n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType);
+ switch( tokenType ){
+ case TK_SPACE: {
+ break;
+ }
+ case TK_ERROR: {
+ sqlite3_free(z);
+ return 0;
+ }
+ case TK_LITERAL: {
+ z[j++] = '?';
+ break;
+ }
+ case TK_PUNCT:
+ case TK_NAME: {
+ if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){
+ if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3]))
+ || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4]))
+ ){
+ /* NULL is a keyword in this case, not a literal value */
+ }else{
+ /* Here the NULL is a literal value */
+ z[j++] = '?';
+ break;
+ }
+ }
+ if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' ';
+ for(k=0; k0 && z[j-1]==' ' ){ j--; }
+ if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; }
+ z[j] = 0;
+
+ /* Make a second pass converting "in(...)" where the "..." is not a
+ ** SELECT statement into "in(?,?,?)" */
+ for(i=0; i5 ){
+ memmove(z+n+5, z+n+k, j-(n+k));
+ }
+ j = j-k+5;
+ z[j] = 0;
+ memcpy(z+n, "?,?,?", 5);
+ }
+ return z;
+}
+
+/*
+** For testing purposes, or to build a stand-alone SQL normalizer program,
+** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link
+** it against any SQLite library. The resulting command-line program will
+** run sqlite3_normalize() over the text of all files named on the command-
+** line and show the result on standard output.
+*/
+#ifdef SQLITE_NORMALIZE_CLI
+#include
+#include
+
+/*
+** Break zIn up into separate SQL statements and run sqlite3_normalize()
+** on each one. Print the result of each run.
+*/
+static void normalizeFile(char *zIn){
+ int i;
+ if( zIn==0 ) return;
+ for(i=0; zIn[i]; i++){
+ char cSaved;
+ if( zIn[i]!=';' ) continue;
+ cSaved = zIn[i+1];
+ zIn[i+1] = 0;
+ if( sqlite3_complete(zIn) ){
+ char *zOut = sqlite3_normalize(zIn);
+ if( zOut ){
+ printf("%s\n", zOut);
+ sqlite3_free(zOut);
+ }else{
+ fprintf(stderr, "ERROR: %s\n", zIn);
+ }
+ zIn[i+1] = cSaved;
+ zIn += i+1;
+ i = -1;
+ }else{
+ zIn[i+1] = cSaved;
+ }
+ }
+}
+
+/*
+** The main routine for "sql_normalize". Read files named on the
+** command-line and run the text of each through sqlite3_normalize().
+*/
+int main(int argc, char **argv){
+ int i;
+ FILE *in;
+ char *zBuf = 0;
+ sqlite3_int64 sz, got;
+
+ for(i=1; ipDest->pMethods->xWrite(p->pDest, pData, p->szPage, iOff);
if( rc!=SQLITE_OK ){
scrubBackupErr(p, "write failed for page %d", pgno);
p->rcErr = SQLITE_IOERR;
}
- if( pgno>p->iLastPage ) p->iLastPage = pgno;
+ if( (u32)pgno>p->iLastPage ) p->iLastPage = pgno;
}
/* Prepare a statement against the "db" database. */
static sqlite3_stmt *scrubBackupPrepare(
ScrubState *p, /* Backup context */
@@ -457,11 +457,11 @@
if( pc > (p->szUsable-4) ){ ln=__LINE__; goto btree_corrupt; }
}
nLocal = K<=X ? K : M;
if( pc+nLocal > p->szUsable-4 ){ ln=__LINE__; goto btree_corrupt; }
iChild = scrubBackupInt32(&a[pc+nLocal]);
- scrubBackupOverflow(p, iChild, P-nLocal);
+ scrubBackupOverflow(p, iChild, (u32)(P-nLocal));
}
/* Walk the right-most tree */
if( aTop[0]==0x05 || aTop[0]==0x02 ){
iChild = scrubBackupInt32(&aTop[8]);
Index: ext/misc/series.c
==================================================================
--- ext/misc/series.c
+++ ext/misc/series.c
@@ -267,10 +267,19 @@
if( idxNum & 4 ){
pCur->iStep = sqlite3_value_int64(argv[i++]);
if( pCur->iStep<1 ) pCur->iStep = 1;
}else{
pCur->iStep = 1;
+ }
+ for(i=0; imnValue = 1;
+ pCur->mxValue = 0;
+ break;
+ }
}
if( idxNum & 8 ){
pCur->isDesc = 1;
pCur->iValue = pCur->mxValue;
if( pCur->iStep>0 ){
@@ -302,48 +311,49 @@
*/
static int seriesBestIndex(
sqlite3_vtab *tab,
sqlite3_index_info *pIdxInfo
){
- int i; /* Loop over constraints */
- int idxNum = 0; /* The query plan bitmask */
- int startIdx = -1; /* Index of the start= constraint, or -1 if none */
- int stopIdx = -1; /* Index of the stop= constraint, or -1 if none */
- int stepIdx = -1; /* Index of the step= constraint, or -1 if none */
- int nArg = 0; /* Number of arguments that seriesFilter() expects */
-
- const struct sqlite3_index_constraint *pConstraint;
+ int i, j; /* Loop over constraints */
+ int idxNum = 0; /* The query plan bitmask */
+ int unusableMask = 0; /* Mask of unusable constraints */
+ int nArg = 0; /* Number of arguments that seriesFilter() expects */
+ int aIdx[3]; /* Constraints on start, stop, and step */
+ const struct sqlite3_index_constraint *pConstraint;
+
+ /* This implementation assumes that the start, stop, and step columns
+ ** are the last three columns in the virtual table. */
+ assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 );
+ assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 );
+ aIdx[0] = aIdx[1] = aIdx[2] = -1;
pConstraint = pIdxInfo->aConstraint;
for(i=0; inConstraint; i++, pConstraint++){
- if( pConstraint->usable==0 ) continue;
- if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
- switch( pConstraint->iColumn ){
- case SERIES_COLUMN_START:
- startIdx = i;
- idxNum |= 1;
- break;
- case SERIES_COLUMN_STOP:
- stopIdx = i;
- idxNum |= 2;
- break;
- case SERIES_COLUMN_STEP:
- stepIdx = i;
- idxNum |= 4;
- break;
- }
- }
- if( startIdx>=0 ){
- pIdxInfo->aConstraintUsage[startIdx].argvIndex = ++nArg;
- pIdxInfo->aConstraintUsage[startIdx].omit= !SQLITE_SERIES_CONSTRAINT_VERIFY;
- }
- if( stopIdx>=0 ){
- pIdxInfo->aConstraintUsage[stopIdx].argvIndex = ++nArg;
- pIdxInfo->aConstraintUsage[stopIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
- }
- if( stepIdx>=0 ){
- pIdxInfo->aConstraintUsage[stepIdx].argvIndex = ++nArg;
- pIdxInfo->aConstraintUsage[stepIdx].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+ int iCol; /* 0 for start, 1 for stop, 2 for step */
+ int iMask; /* bitmask for those column */
+ if( pConstraint->iColumniColumn - SERIES_COLUMN_START;
+ assert( iCol>=0 && iCol<=2 );
+ iMask = 1 << iCol;
+ if( pConstraint->usable==0 ){
+ unusableMask |= iMask;
+ continue;
+ }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ idxNum |= iMask;
+ aIdx[iCol] = i;
+ }
+ }
+ for(i=0; i<3; i++){
+ if( (j = aIdx[i])>=0 ){
+ pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg;
+ pIdxInfo->aConstraintUsage[j].omit = !SQLITE_SERIES_CONSTRAINT_VERIFY;
+ }
+ }
+ if( (unusableMask & ~idxNum)!=0 ){
+ /* The start, stop, and step columns are inputs. Therefore if there
+ ** are unusable constraints on any of start, stop, or step then
+ ** this plan is unusable */
+ return SQLITE_CONSTRAINT;
}
if( (idxNum & 3)==3 ){
/* Both start= and stop= boundaries are available. This is the
** the preferred case */
pIdxInfo->estimatedCost = (double)(2 - ((idxNum&4)!=0));
@@ -354,11 +364,10 @@
}
}else{
/* If either boundary is missing, we have to generate a huge span
** of numbers. Make this case very expensive so that the query
** planner will work hard to avoid it. */
- pIdxInfo->estimatedCost = (double)2147483647;
pIdxInfo->estimatedRows = 2147483647;
}
pIdxInfo->idxNum = idxNum;
return SQLITE_OK;
}
Index: ext/misc/sha1.c
==================================================================
--- ext/misc/sha1.c
+++ ext/misc/sha1.c
@@ -8,11 +8,11 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA1 hashes.
** Two SQL functions are implemented:
**
** sha1(X)
** sha1_query(Y)
**
Index: ext/misc/shathree.c
==================================================================
--- ext/misc/shathree.c
+++ ext/misc/shathree.c
@@ -8,11 +8,11 @@
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
-** This SQLite extension implements a functions that compute SHA1 hashes.
+** This SQLite extension implements functions that compute SHA3 hashes.
** Two SQL functions are implemented:
**
** sha3(X,SIZE)
** sha3_query(Y,SIZE)
**
@@ -76,13 +76,13 @@
/*
** A single step of the Keccak mixing function for a 1600-bit state
*/
static void KeccakF1600Step(SHA3Context *p){
int i;
- u64 B0, B1, B2, B3, B4;
- u64 C0, C1, C2, C3, C4;
- u64 D0, D1, D2, D3, D4;
+ u64 b0, b1, b2, b3, b4;
+ u64 c0, c1, c2, c3, c4;
+ u64 d0, d1, d2, d3, d4;
static const u64 RC[] = {
0x0000000000000001ULL, 0x0000000000008082ULL,
0x800000000000808aULL, 0x8000000080008000ULL,
0x000000000000808bULL, 0x0000000080000001ULL,
0x8000000080008081ULL, 0x8000000000008009ULL,
@@ -93,305 +93,305 @@
0x8000000000008002ULL, 0x8000000000000080ULL,
0x000000000000800aULL, 0x800000008000000aULL,
0x8000000080008081ULL, 0x8000000000008080ULL,
0x0000000080000001ULL, 0x8000000080008008ULL
};
-# define A00 (p->u.s[0])
-# define A01 (p->u.s[1])
-# define A02 (p->u.s[2])
-# define A03 (p->u.s[3])
-# define A04 (p->u.s[4])
-# define A10 (p->u.s[5])
-# define A11 (p->u.s[6])
-# define A12 (p->u.s[7])
-# define A13 (p->u.s[8])
-# define A14 (p->u.s[9])
-# define A20 (p->u.s[10])
-# define A21 (p->u.s[11])
-# define A22 (p->u.s[12])
-# define A23 (p->u.s[13])
-# define A24 (p->u.s[14])
-# define A30 (p->u.s[15])
-# define A31 (p->u.s[16])
-# define A32 (p->u.s[17])
-# define A33 (p->u.s[18])
-# define A34 (p->u.s[19])
-# define A40 (p->u.s[20])
-# define A41 (p->u.s[21])
-# define A42 (p->u.s[22])
-# define A43 (p->u.s[23])
-# define A44 (p->u.s[24])
+# define a00 (p->u.s[0])
+# define a01 (p->u.s[1])
+# define a02 (p->u.s[2])
+# define a03 (p->u.s[3])
+# define a04 (p->u.s[4])
+# define a10 (p->u.s[5])
+# define a11 (p->u.s[6])
+# define a12 (p->u.s[7])
+# define a13 (p->u.s[8])
+# define a14 (p->u.s[9])
+# define a20 (p->u.s[10])
+# define a21 (p->u.s[11])
+# define a22 (p->u.s[12])
+# define a23 (p->u.s[13])
+# define a24 (p->u.s[14])
+# define a30 (p->u.s[15])
+# define a31 (p->u.s[16])
+# define a32 (p->u.s[17])
+# define a33 (p->u.s[18])
+# define a34 (p->u.s[19])
+# define a40 (p->u.s[20])
+# define a41 (p->u.s[21])
+# define a42 (p->u.s[22])
+# define a43 (p->u.s[23])
+# define a44 (p->u.s[24])
# define ROL64(a,x) ((a<>(64-x)))
for(i=0; i<24; i+=4){
- C0 = A00^A10^A20^A30^A40;
- C1 = A01^A11^A21^A31^A41;
- C2 = A02^A12^A22^A32^A42;
- C3 = A03^A13^A23^A33^A43;
- C4 = A04^A14^A24^A34^A44;
- D0 = C4^ROL64(C1, 1);
- D1 = C0^ROL64(C2, 1);
- D2 = C1^ROL64(C3, 1);
- D3 = C2^ROL64(C4, 1);
- D4 = C3^ROL64(C0, 1);
-
- B0 = (A00^D0);
- B1 = ROL64((A11^D1), 44);
- B2 = ROL64((A22^D2), 43);
- B3 = ROL64((A33^D3), 21);
- B4 = ROL64((A44^D4), 14);
- A00 = B0 ^((~B1)& B2 );
- A00 ^= RC[i];
- A11 = B1 ^((~B2)& B3 );
- A22 = B2 ^((~B3)& B4 );
- A33 = B3 ^((~B4)& B0 );
- A44 = B4 ^((~B0)& B1 );
-
- B2 = ROL64((A20^D0), 3);
- B3 = ROL64((A31^D1), 45);
- B4 = ROL64((A42^D2), 61);
- B0 = ROL64((A03^D3), 28);
- B1 = ROL64((A14^D4), 20);
- A20 = B0 ^((~B1)& B2 );
- A31 = B1 ^((~B2)& B3 );
- A42 = B2 ^((~B3)& B4 );
- A03 = B3 ^((~B4)& B0 );
- A14 = B4 ^((~B0)& B1 );
-
- B4 = ROL64((A40^D0), 18);
- B0 = ROL64((A01^D1), 1);
- B1 = ROL64((A12^D2), 6);
- B2 = ROL64((A23^D3), 25);
- B3 = ROL64((A34^D4), 8);
- A40 = B0 ^((~B1)& B2 );
- A01 = B1 ^((~B2)& B3 );
- A12 = B2 ^((~B3)& B4 );
- A23 = B3 ^((~B4)& B0 );
- A34 = B4 ^((~B0)& B1 );
-
- B1 = ROL64((A10^D0), 36);
- B2 = ROL64((A21^D1), 10);
- B3 = ROL64((A32^D2), 15);
- B4 = ROL64((A43^D3), 56);
- B0 = ROL64((A04^D4), 27);
- A10 = B0 ^((~B1)& B2 );
- A21 = B1 ^((~B2)& B3 );
- A32 = B2 ^((~B3)& B4 );
- A43 = B3 ^((~B4)& B0 );
- A04 = B4 ^((~B0)& B1 );
-
- B3 = ROL64((A30^D0), 41);
- B4 = ROL64((A41^D1), 2);
- B0 = ROL64((A02^D2), 62);
- B1 = ROL64((A13^D3), 55);
- B2 = ROL64((A24^D4), 39);
- A30 = B0 ^((~B1)& B2 );
- A41 = B1 ^((~B2)& B3 );
- A02 = B2 ^((~B3)& B4 );
- A13 = B3 ^((~B4)& B0 );
- A24 = B4 ^((~B0)& B1 );
-
- C0 = A00^A20^A40^A10^A30;
- C1 = A11^A31^A01^A21^A41;
- C2 = A22^A42^A12^A32^A02;
- C3 = A33^A03^A23^A43^A13;
- C4 = A44^A14^A34^A04^A24;
- D0 = C4^ROL64(C1, 1);
- D1 = C0^ROL64(C2, 1);
- D2 = C1^ROL64(C3, 1);
- D3 = C2^ROL64(C4, 1);
- D4 = C3^ROL64(C0, 1);
-
- B0 = (A00^D0);
- B1 = ROL64((A31^D1), 44);
- B2 = ROL64((A12^D2), 43);
- B3 = ROL64((A43^D3), 21);
- B4 = ROL64((A24^D4), 14);
- A00 = B0 ^((~B1)& B2 );
- A00 ^= RC[i+1];
- A31 = B1 ^((~B2)& B3 );
- A12 = B2 ^((~B3)& B4 );
- A43 = B3 ^((~B4)& B0 );
- A24 = B4 ^((~B0)& B1 );
-
- B2 = ROL64((A40^D0), 3);
- B3 = ROL64((A21^D1), 45);
- B4 = ROL64((A02^D2), 61);
- B0 = ROL64((A33^D3), 28);
- B1 = ROL64((A14^D4), 20);
- A40 = B0 ^((~B1)& B2 );
- A21 = B1 ^((~B2)& B3 );
- A02 = B2 ^((~B3)& B4 );
- A33 = B3 ^((~B4)& B0 );
- A14 = B4 ^((~B0)& B1 );
-
- B4 = ROL64((A30^D0), 18);
- B0 = ROL64((A11^D1), 1);
- B1 = ROL64((A42^D2), 6);
- B2 = ROL64((A23^D3), 25);
- B3 = ROL64((A04^D4), 8);
- A30 = B0 ^((~B1)& B2 );
- A11 = B1 ^((~B2)& B3 );
- A42 = B2 ^((~B3)& B4 );
- A23 = B3 ^((~B4)& B0 );
- A04 = B4 ^((~B0)& B1 );
-
- B1 = ROL64((A20^D0), 36);
- B2 = ROL64((A01^D1), 10);
- B3 = ROL64((A32^D2), 15);
- B4 = ROL64((A13^D3), 56);
- B0 = ROL64((A44^D4), 27);
- A20 = B0 ^((~B1)& B2 );
- A01 = B1 ^((~B2)& B3 );
- A32 = B2 ^((~B3)& B4 );
- A13 = B3 ^((~B4)& B0 );
- A44 = B4 ^((~B0)& B1 );
-
- B3 = ROL64((A10^D0), 41);
- B4 = ROL64((A41^D1), 2);
- B0 = ROL64((A22^D2), 62);
- B1 = ROL64((A03^D3), 55);
- B2 = ROL64((A34^D4), 39);
- A10 = B0 ^((~B1)& B2 );
- A41 = B1 ^((~B2)& B3 );
- A22 = B2 ^((~B3)& B4 );
- A03 = B3 ^((~B4)& B0 );
- A34 = B4 ^((~B0)& B1 );
-
- C0 = A00^A40^A30^A20^A10;
- C1 = A31^A21^A11^A01^A41;
- C2 = A12^A02^A42^A32^A22;
- C3 = A43^A33^A23^A13^A03;
- C4 = A24^A14^A04^A44^A34;
- D0 = C4^ROL64(C1, 1);
- D1 = C0^ROL64(C2, 1);
- D2 = C1^ROL64(C3, 1);
- D3 = C2^ROL64(C4, 1);
- D4 = C3^ROL64(C0, 1);
-
- B0 = (A00^D0);
- B1 = ROL64((A21^D1), 44);
- B2 = ROL64((A42^D2), 43);
- B3 = ROL64((A13^D3), 21);
- B4 = ROL64((A34^D4), 14);
- A00 = B0 ^((~B1)& B2 );
- A00 ^= RC[i+2];
- A21 = B1 ^((~B2)& B3 );
- A42 = B2 ^((~B3)& B4 );
- A13 = B3 ^((~B4)& B0 );
- A34 = B4 ^((~B0)& B1 );
-
- B2 = ROL64((A30^D0), 3);
- B3 = ROL64((A01^D1), 45);
- B4 = ROL64((A22^D2), 61);
- B0 = ROL64((A43^D3), 28);
- B1 = ROL64((A14^D4), 20);
- A30 = B0 ^((~B1)& B2 );
- A01 = B1 ^((~B2)& B3 );
- A22 = B2 ^((~B3)& B4 );
- A43 = B3 ^((~B4)& B0 );
- A14 = B4 ^((~B0)& B1 );
-
- B4 = ROL64((A10^D0), 18);
- B0 = ROL64((A31^D1), 1);
- B1 = ROL64((A02^D2), 6);
- B2 = ROL64((A23^D3), 25);
- B3 = ROL64((A44^D4), 8);
- A10 = B0 ^((~B1)& B2 );
- A31 = B1 ^((~B2)& B3 );
- A02 = B2 ^((~B3)& B4 );
- A23 = B3 ^((~B4)& B0 );
- A44 = B4 ^((~B0)& B1 );
-
- B1 = ROL64((A40^D0), 36);
- B2 = ROL64((A11^D1), 10);
- B3 = ROL64((A32^D2), 15);
- B4 = ROL64((A03^D3), 56);
- B0 = ROL64((A24^D4), 27);
- A40 = B0 ^((~B1)& B2 );
- A11 = B1 ^((~B2)& B3 );
- A32 = B2 ^((~B3)& B4 );
- A03 = B3 ^((~B4)& B0 );
- A24 = B4 ^((~B0)& B1 );
-
- B3 = ROL64((A20^D0), 41);
- B4 = ROL64((A41^D1), 2);
- B0 = ROL64((A12^D2), 62);
- B1 = ROL64((A33^D3), 55);
- B2 = ROL64((A04^D4), 39);
- A20 = B0 ^((~B1)& B2 );
- A41 = B1 ^((~B2)& B3 );
- A12 = B2 ^((~B3)& B4 );
- A33 = B3 ^((~B4)& B0 );
- A04 = B4 ^((~B0)& B1 );
-
- C0 = A00^A30^A10^A40^A20;
- C1 = A21^A01^A31^A11^A41;
- C2 = A42^A22^A02^A32^A12;
- C3 = A13^A43^A23^A03^A33;
- C4 = A34^A14^A44^A24^A04;
- D0 = C4^ROL64(C1, 1);
- D1 = C0^ROL64(C2, 1);
- D2 = C1^ROL64(C3, 1);
- D3 = C2^ROL64(C4, 1);
- D4 = C3^ROL64(C0, 1);
-
- B0 = (A00^D0);
- B1 = ROL64((A01^D1), 44);
- B2 = ROL64((A02^D2), 43);
- B3 = ROL64((A03^D3), 21);
- B4 = ROL64((A04^D4), 14);
- A00 = B0 ^((~B1)& B2 );
- A00 ^= RC[i+3];
- A01 = B1 ^((~B2)& B3 );
- A02 = B2 ^((~B3)& B4 );
- A03 = B3 ^((~B4)& B0 );
- A04 = B4 ^((~B0)& B1 );
-
- B2 = ROL64((A10^D0), 3);
- B3 = ROL64((A11^D1), 45);
- B4 = ROL64((A12^D2), 61);
- B0 = ROL64((A13^D3), 28);
- B1 = ROL64((A14^D4), 20);
- A10 = B0 ^((~B1)& B2 );
- A11 = B1 ^((~B2)& B3 );
- A12 = B2 ^((~B3)& B4 );
- A13 = B3 ^((~B4)& B0 );
- A14 = B4 ^((~B0)& B1 );
-
- B4 = ROL64((A20^D0), 18);
- B0 = ROL64((A21^D1), 1);
- B1 = ROL64((A22^D2), 6);
- B2 = ROL64((A23^D3), 25);
- B3 = ROL64((A24^D4), 8);
- A20 = B0 ^((~B1)& B2 );
- A21 = B1 ^((~B2)& B3 );
- A22 = B2 ^((~B3)& B4 );
- A23 = B3 ^((~B4)& B0 );
- A24 = B4 ^((~B0)& B1 );
-
- B1 = ROL64((A30^D0), 36);
- B2 = ROL64((A31^D1), 10);
- B3 = ROL64((A32^D2), 15);
- B4 = ROL64((A33^D3), 56);
- B0 = ROL64((A34^D4), 27);
- A30 = B0 ^((~B1)& B2 );
- A31 = B1 ^((~B2)& B3 );
- A32 = B2 ^((~B3)& B4 );
- A33 = B3 ^((~B4)& B0 );
- A34 = B4 ^((~B0)& B1 );
-
- B3 = ROL64((A40^D0), 41);
- B4 = ROL64((A41^D1), 2);
- B0 = ROL64((A42^D2), 62);
- B1 = ROL64((A43^D3), 55);
- B2 = ROL64((A44^D4), 39);
- A40 = B0 ^((~B1)& B2 );
- A41 = B1 ^((~B2)& B3 );
- A42 = B2 ^((~B3)& B4 );
- A43 = B3 ^((~B4)& B0 );
- A44 = B4 ^((~B0)& B1 );
+ c0 = a00^a10^a20^a30^a40;
+ c1 = a01^a11^a21^a31^a41;
+ c2 = a02^a12^a22^a32^a42;
+ c3 = a03^a13^a23^a33^a43;
+ c4 = a04^a14^a24^a34^a44;
+ d0 = c4^ROL64(c1, 1);
+ d1 = c0^ROL64(c2, 1);
+ d2 = c1^ROL64(c3, 1);
+ d3 = c2^ROL64(c4, 1);
+ d4 = c3^ROL64(c0, 1);
+
+ b0 = (a00^d0);
+ b1 = ROL64((a11^d1), 44);
+ b2 = ROL64((a22^d2), 43);
+ b3 = ROL64((a33^d3), 21);
+ b4 = ROL64((a44^d4), 14);
+ a00 = b0 ^((~b1)& b2 );
+ a00 ^= RC[i];
+ a11 = b1 ^((~b2)& b3 );
+ a22 = b2 ^((~b3)& b4 );
+ a33 = b3 ^((~b4)& b0 );
+ a44 = b4 ^((~b0)& b1 );
+
+ b2 = ROL64((a20^d0), 3);
+ b3 = ROL64((a31^d1), 45);
+ b4 = ROL64((a42^d2), 61);
+ b0 = ROL64((a03^d3), 28);
+ b1 = ROL64((a14^d4), 20);
+ a20 = b0 ^((~b1)& b2 );
+ a31 = b1 ^((~b2)& b3 );
+ a42 = b2 ^((~b3)& b4 );
+ a03 = b3 ^((~b4)& b0 );
+ a14 = b4 ^((~b0)& b1 );
+
+ b4 = ROL64((a40^d0), 18);
+ b0 = ROL64((a01^d1), 1);
+ b1 = ROL64((a12^d2), 6);
+ b2 = ROL64((a23^d3), 25);
+ b3 = ROL64((a34^d4), 8);
+ a40 = b0 ^((~b1)& b2 );
+ a01 = b1 ^((~b2)& b3 );
+ a12 = b2 ^((~b3)& b4 );
+ a23 = b3 ^((~b4)& b0 );
+ a34 = b4 ^((~b0)& b1 );
+
+ b1 = ROL64((a10^d0), 36);
+ b2 = ROL64((a21^d1), 10);
+ b3 = ROL64((a32^d2), 15);
+ b4 = ROL64((a43^d3), 56);
+ b0 = ROL64((a04^d4), 27);
+ a10 = b0 ^((~b1)& b2 );
+ a21 = b1 ^((~b2)& b3 );
+ a32 = b2 ^((~b3)& b4 );
+ a43 = b3 ^((~b4)& b0 );
+ a04 = b4 ^((~b0)& b1 );
+
+ b3 = ROL64((a30^d0), 41);
+ b4 = ROL64((a41^d1), 2);
+ b0 = ROL64((a02^d2), 62);
+ b1 = ROL64((a13^d3), 55);
+ b2 = ROL64((a24^d4), 39);
+ a30 = b0 ^((~b1)& b2 );
+ a41 = b1 ^((~b2)& b3 );
+ a02 = b2 ^((~b3)& b4 );
+ a13 = b3 ^((~b4)& b0 );
+ a24 = b4 ^((~b0)& b1 );
+
+ c0 = a00^a20^a40^a10^a30;
+ c1 = a11^a31^a01^a21^a41;
+ c2 = a22^a42^a12^a32^a02;
+ c3 = a33^a03^a23^a43^a13;
+ c4 = a44^a14^a34^a04^a24;
+ d0 = c4^ROL64(c1, 1);
+ d1 = c0^ROL64(c2, 1);
+ d2 = c1^ROL64(c3, 1);
+ d3 = c2^ROL64(c4, 1);
+ d4 = c3^ROL64(c0, 1);
+
+ b0 = (a00^d0);
+ b1 = ROL64((a31^d1), 44);
+ b2 = ROL64((a12^d2), 43);
+ b3 = ROL64((a43^d3), 21);
+ b4 = ROL64((a24^d4), 14);
+ a00 = b0 ^((~b1)& b2 );
+ a00 ^= RC[i+1];
+ a31 = b1 ^((~b2)& b3 );
+ a12 = b2 ^((~b3)& b4 );
+ a43 = b3 ^((~b4)& b0 );
+ a24 = b4 ^((~b0)& b1 );
+
+ b2 = ROL64((a40^d0), 3);
+ b3 = ROL64((a21^d1), 45);
+ b4 = ROL64((a02^d2), 61);
+ b0 = ROL64((a33^d3), 28);
+ b1 = ROL64((a14^d4), 20);
+ a40 = b0 ^((~b1)& b2 );
+ a21 = b1 ^((~b2)& b3 );
+ a02 = b2 ^((~b3)& b4 );
+ a33 = b3 ^((~b4)& b0 );
+ a14 = b4 ^((~b0)& b1 );
+
+ b4 = ROL64((a30^d0), 18);
+ b0 = ROL64((a11^d1), 1);
+ b1 = ROL64((a42^d2), 6);
+ b2 = ROL64((a23^d3), 25);
+ b3 = ROL64((a04^d4), 8);
+ a30 = b0 ^((~b1)& b2 );
+ a11 = b1 ^((~b2)& b3 );
+ a42 = b2 ^((~b3)& b4 );
+ a23 = b3 ^((~b4)& b0 );
+ a04 = b4 ^((~b0)& b1 );
+
+ b1 = ROL64((a20^d0), 36);
+ b2 = ROL64((a01^d1), 10);
+ b3 = ROL64((a32^d2), 15);
+ b4 = ROL64((a13^d3), 56);
+ b0 = ROL64((a44^d4), 27);
+ a20 = b0 ^((~b1)& b2 );
+ a01 = b1 ^((~b2)& b3 );
+ a32 = b2 ^((~b3)& b4 );
+ a13 = b3 ^((~b4)& b0 );
+ a44 = b4 ^((~b0)& b1 );
+
+ b3 = ROL64((a10^d0), 41);
+ b4 = ROL64((a41^d1), 2);
+ b0 = ROL64((a22^d2), 62);
+ b1 = ROL64((a03^d3), 55);
+ b2 = ROL64((a34^d4), 39);
+ a10 = b0 ^((~b1)& b2 );
+ a41 = b1 ^((~b2)& b3 );
+ a22 = b2 ^((~b3)& b4 );
+ a03 = b3 ^((~b4)& b0 );
+ a34 = b4 ^((~b0)& b1 );
+
+ c0 = a00^a40^a30^a20^a10;
+ c1 = a31^a21^a11^a01^a41;
+ c2 = a12^a02^a42^a32^a22;
+ c3 = a43^a33^a23^a13^a03;
+ c4 = a24^a14^a04^a44^a34;
+ d0 = c4^ROL64(c1, 1);
+ d1 = c0^ROL64(c2, 1);
+ d2 = c1^ROL64(c3, 1);
+ d3 = c2^ROL64(c4, 1);
+ d4 = c3^ROL64(c0, 1);
+
+ b0 = (a00^d0);
+ b1 = ROL64((a21^d1), 44);
+ b2 = ROL64((a42^d2), 43);
+ b3 = ROL64((a13^d3), 21);
+ b4 = ROL64((a34^d4), 14);
+ a00 = b0 ^((~b1)& b2 );
+ a00 ^= RC[i+2];
+ a21 = b1 ^((~b2)& b3 );
+ a42 = b2 ^((~b3)& b4 );
+ a13 = b3 ^((~b4)& b0 );
+ a34 = b4 ^((~b0)& b1 );
+
+ b2 = ROL64((a30^d0), 3);
+ b3 = ROL64((a01^d1), 45);
+ b4 = ROL64((a22^d2), 61);
+ b0 = ROL64((a43^d3), 28);
+ b1 = ROL64((a14^d4), 20);
+ a30 = b0 ^((~b1)& b2 );
+ a01 = b1 ^((~b2)& b3 );
+ a22 = b2 ^((~b3)& b4 );
+ a43 = b3 ^((~b4)& b0 );
+ a14 = b4 ^((~b0)& b1 );
+
+ b4 = ROL64((a10^d0), 18);
+ b0 = ROL64((a31^d1), 1);
+ b1 = ROL64((a02^d2), 6);
+ b2 = ROL64((a23^d3), 25);
+ b3 = ROL64((a44^d4), 8);
+ a10 = b0 ^((~b1)& b2 );
+ a31 = b1 ^((~b2)& b3 );
+ a02 = b2 ^((~b3)& b4 );
+ a23 = b3 ^((~b4)& b0 );
+ a44 = b4 ^((~b0)& b1 );
+
+ b1 = ROL64((a40^d0), 36);
+ b2 = ROL64((a11^d1), 10);
+ b3 = ROL64((a32^d2), 15);
+ b4 = ROL64((a03^d3), 56);
+ b0 = ROL64((a24^d4), 27);
+ a40 = b0 ^((~b1)& b2 );
+ a11 = b1 ^((~b2)& b3 );
+ a32 = b2 ^((~b3)& b4 );
+ a03 = b3 ^((~b4)& b0 );
+ a24 = b4 ^((~b0)& b1 );
+
+ b3 = ROL64((a20^d0), 41);
+ b4 = ROL64((a41^d1), 2);
+ b0 = ROL64((a12^d2), 62);
+ b1 = ROL64((a33^d3), 55);
+ b2 = ROL64((a04^d4), 39);
+ a20 = b0 ^((~b1)& b2 );
+ a41 = b1 ^((~b2)& b3 );
+ a12 = b2 ^((~b3)& b4 );
+ a33 = b3 ^((~b4)& b0 );
+ a04 = b4 ^((~b0)& b1 );
+
+ c0 = a00^a30^a10^a40^a20;
+ c1 = a21^a01^a31^a11^a41;
+ c2 = a42^a22^a02^a32^a12;
+ c3 = a13^a43^a23^a03^a33;
+ c4 = a34^a14^a44^a24^a04;
+ d0 = c4^ROL64(c1, 1);
+ d1 = c0^ROL64(c2, 1);
+ d2 = c1^ROL64(c3, 1);
+ d3 = c2^ROL64(c4, 1);
+ d4 = c3^ROL64(c0, 1);
+
+ b0 = (a00^d0);
+ b1 = ROL64((a01^d1), 44);
+ b2 = ROL64((a02^d2), 43);
+ b3 = ROL64((a03^d3), 21);
+ b4 = ROL64((a04^d4), 14);
+ a00 = b0 ^((~b1)& b2 );
+ a00 ^= RC[i+3];
+ a01 = b1 ^((~b2)& b3 );
+ a02 = b2 ^((~b3)& b4 );
+ a03 = b3 ^((~b4)& b0 );
+ a04 = b4 ^((~b0)& b1 );
+
+ b2 = ROL64((a10^d0), 3);
+ b3 = ROL64((a11^d1), 45);
+ b4 = ROL64((a12^d2), 61);
+ b0 = ROL64((a13^d3), 28);
+ b1 = ROL64((a14^d4), 20);
+ a10 = b0 ^((~b1)& b2 );
+ a11 = b1 ^((~b2)& b3 );
+ a12 = b2 ^((~b3)& b4 );
+ a13 = b3 ^((~b4)& b0 );
+ a14 = b4 ^((~b0)& b1 );
+
+ b4 = ROL64((a20^d0), 18);
+ b0 = ROL64((a21^d1), 1);
+ b1 = ROL64((a22^d2), 6);
+ b2 = ROL64((a23^d3), 25);
+ b3 = ROL64((a24^d4), 8);
+ a20 = b0 ^((~b1)& b2 );
+ a21 = b1 ^((~b2)& b3 );
+ a22 = b2 ^((~b3)& b4 );
+ a23 = b3 ^((~b4)& b0 );
+ a24 = b4 ^((~b0)& b1 );
+
+ b1 = ROL64((a30^d0), 36);
+ b2 = ROL64((a31^d1), 10);
+ b3 = ROL64((a32^d2), 15);
+ b4 = ROL64((a33^d3), 56);
+ b0 = ROL64((a34^d4), 27);
+ a30 = b0 ^((~b1)& b2 );
+ a31 = b1 ^((~b2)& b3 );
+ a32 = b2 ^((~b3)& b4 );
+ a33 = b3 ^((~b4)& b0 );
+ a34 = b4 ^((~b0)& b1 );
+
+ b3 = ROL64((a40^d0), 41);
+ b4 = ROL64((a41^d1), 2);
+ b0 = ROL64((a42^d2), 62);
+ b1 = ROL64((a43^d3), 55);
+ b2 = ROL64((a44^d4), 39);
+ a40 = b0 ^((~b1)& b2 );
+ a41 = b1 ^((~b2)& b3 );
+ a42 = b2 ^((~b3)& b4 );
+ a43 = b3 ^((~b4)& b0 );
+ a44 = b4 ^((~b0)& b1 );
}
}
/*
** Initialize a new hash. iSize determines the size of the hash
Index: ext/misc/spellfix.c
==================================================================
--- ext/misc/spellfix.c
+++ ext/misc/spellfix.c
@@ -16,10 +16,16 @@
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#ifndef SQLITE_AMALGAMATION
+# if !defined(NDEBUG) && !defined(SQLITE_DEBUG)
+# define NDEBUG 1
+# endif
+# if defined(NDEBUG) && defined(SQLITE_DEBUG)
+# undef NDEBUG
+# endif
# include
# include
# include
# include
# define ALWAYS(X) 1
@@ -649,10 +655,83 @@
static void editDist3ConfigDelete(void *pIn){
EditDist3Config *p = (EditDist3Config*)pIn;
editDist3ConfigClear(p);
sqlite3_free(p);
}
+
+/* Compare the FROM values of two EditDist3Cost objects, for sorting.
+** Return negative, zero, or positive if the A is less than, equal to,
+** or greater than B.
+*/
+static int editDist3CostCompare(EditDist3Cost *pA, EditDist3Cost *pB){
+ int n = pA->nFrom;
+ int rc;
+ if( n>pB->nFrom ) n = pB->nFrom;
+ rc = strncmp(pA->a, pB->a, n);
+ if( rc==0 ) rc = pA->nFrom - pB->nFrom;
+ return rc;
+}
+
+/*
+** Merge together two sorted lists of EditDist3Cost objects, in order
+** of increasing FROM.
+*/
+static EditDist3Cost *editDist3CostMerge(
+ EditDist3Cost *pA,
+ EditDist3Cost *pB
+){
+ EditDist3Cost *pHead = 0;
+ EditDist3Cost **ppTail = &pHead;
+ EditDist3Cost *p;
+ while( pA && pB ){
+ if( editDist3CostCompare(pA,pB)<=0 ){
+ p = pA;
+ pA = pA->pNext;
+ }else{
+ p = pB;
+ pB = pB->pNext;
+ }
+ *ppTail = p;
+ ppTail = &p->pNext;
+ }
+ if( pA ){
+ *ppTail = pA;
+ }else{
+ *ppTail = pB;
+ }
+ return pHead;
+}
+
+/*
+** Sort a list of EditDist3Cost objects into order of increasing FROM
+*/
+static EditDist3Cost *editDist3CostSort(EditDist3Cost *pList){
+ EditDist3Cost *ap[60], *p;
+ int i;
+ int mx = 0;
+ ap[0] = 0;
+ ap[1] = 0;
+ while( pList ){
+ p = pList;
+ pList = p->pNext;
+ p->pNext = 0;
+ for(i=0; ap[i]; i++){
+ p = editDist3CostMerge(ap[i],p);
+ ap[i] = 0;
+ }
+ ap[i] = p;
+ if( i>mx ){
+ mx = i;
+ ap[i+1] = 0;
+ }
+ }
+ p = 0;
+ for(i=0; i<=mx; i++){
+ if( ap[i] ) p = editDist3CostMerge(p,ap[i]);
+ }
+ return p;
+}
/*
** Load all edit-distance weights from a table.
*/
static int editDist3ConfigLoad(
@@ -683,10 +762,11 @@
assert( zFrom!=0 || nFrom==0 );
assert( zTo!=0 || nTo==0 );
if( nFrom>100 || nTo>100 ) continue;
if( iCost<0 ) continue;
+ if( iCost>=10000 ) continue; /* Costs above 10K are considered infinite */
if( pLang==0 || iLang!=iLangPrev ){
EditDist3Lang *pNew;
pNew = sqlite3_realloc64(p->a, (p->nLang+1)*sizeof(p->a[0]));
if( pNew==0 ){ rc = SQLITE_NOMEM; break; }
p->a = pNew;
@@ -720,10 +800,16 @@
pLang->pCost = pCost;
}
}
rc2 = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK ) rc = rc2;
+ if( rc==SQLITE_OK ){
+ int iLang;
+ for(iLang=0; iLangnLang; iLang++){
+ p->a[iLang].pCost = editDist3CostSort(p->a[iLang].pCost);
+ }
+ }
return rc;
}
/*
** Return the length (in bytes) of a utf-8 character. Or return a maximum
@@ -747,10 +833,12 @@
/*
** Return TRUE (non-zero) if the To side of the given cost matches
** the given string.
*/
static int matchTo(EditDist3Cost *p, const char *z, int n){
+ assert( n>0 );
+ if( p->a[p->nFrom]!=z[0] ) return 0;
if( p->nTo>n ) return 0;
if( strncmp(p->a+p->nFrom, z, p->nTo)!=0 ) return 0;
return 1;
}
@@ -758,11 +846,14 @@
** Return TRUE (non-zero) if the From side of the given cost matches
** the given string.
*/
static int matchFrom(EditDist3Cost *p, const char *z, int n){
assert( p->nFrom<=n );
- if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
+ if( p->nFrom ){
+ if( p->a[0]!=z[0] ) return 0;
+ if( strncmp(p->a, z, p->nFrom)!=0 ) return 0;
+ }
return 1;
}
/*
** Return TRUE (non-zero) of the next FROM character and the next TO
@@ -774,11 +865,13 @@
const char *z2, /* Right-handl comparison character */
int n2 /* Bytes remaining in z2[] */
){
int b1 = pStr->a[n1].nByte;
if( b1>n2 ) return 0;
- if( memcmp(pStr->z+n1, z2, b1)!=0 ) return 0;
+ assert( b1>0 );
+ if( pStr->z[n1]!=z2[0] ) return 0;
+ if( strncmp(pStr->z+n1, z2, b1)!=0 ) return 0;
return 1;
}
/*
** Delete an EditDist3FromString objecct
@@ -856,25 +949,22 @@
}
/*
** Update entry m[i] such that it is the minimum of its current value
** and m[j]+iCost.
-**
-** If the iCost is 1,000,000 or greater, then consider the cost to be
-** infinite and skip the update.
*/
static void updateCost(
unsigned int *m,
int i,
int j,
int iCost
){
+ unsigned int b;
assert( iCost>=0 );
- if( iCost<10000 ){
- unsigned int b = m[j] + iCost;
- if( bpCost; p; p=p->pNext){
EditDist3Cost **apNew;
- if( p->nFrom>0 ) continue;
+ if( p->nFrom>0 ) break;
if( i2+p->nTo>n2 ) continue;
+ if( p->a[0]>z2[i2] ) break;
if( matchTo(p, z2+i2, n2-i2)==0 ) continue;
a2[i2].nIns++;
apNew = sqlite3_realloc64(a2[i2].apIns, sizeof(*apNew)*a2[i2].nIns);
if( apNew==0 ){
res = -1; /* Out of memory */
@@ -1120,19 +1211,21 @@
int rc;
EditDist3Config *pConfig = sqlite3_malloc64( sizeof(*pConfig) );
if( pConfig==0 ) return SQLITE_NOMEM;
memset(pConfig, 0, sizeof(*pConfig));
rc = sqlite3_create_function_v2(db, "editdist3",
- 2, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0);
+ 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+ editDist3SqlFunc, 0, 0, 0);
if( rc==SQLITE_OK ){
rc = sqlite3_create_function_v2(db, "editdist3",
- 3, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0);
+ 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+ editDist3SqlFunc, 0, 0, 0);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_function_v2(db, "editdist3",
- 1, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0,
- editDist3ConfigDelete);
+ 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig,
+ editDist3SqlFunc, 0, 0, editDist3ConfigDelete);
}else{
sqlite3_free(pConfig);
}
return rc;
}
@@ -1196,407 +1289,418 @@
i += sz;
}
return nChar;
}
+typedef struct Transliteration Transliteration;
+struct Transliteration {
+ unsigned short int cFrom;
+ unsigned char cTo0, cTo1, cTo2, cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+ unsigned char cTo4;
+#endif
+};
+
/*
** Table of translations from unicode characters into ASCII.
*/
-static const struct {
- unsigned short int cFrom;
- unsigned char cTo0, cTo1;
-} translit[] = {
- { 0x00A0, 0x20, 0x00 }, /* to */
- { 0x00B5, 0x75, 0x00 }, /* µ to u */
- { 0x00C0, 0x41, 0x00 }, /* À to A */
- { 0x00C1, 0x41, 0x00 }, /* Á to A */
- { 0x00C2, 0x41, 0x00 }, /* Â to A */
- { 0x00C3, 0x41, 0x00 }, /* Ã to A */
- { 0x00C4, 0x41, 0x65 }, /* Ä to Ae */
- { 0x00C5, 0x41, 0x61 }, /* Å to Aa */
- { 0x00C6, 0x41, 0x45 }, /* Æ to AE */
- { 0x00C7, 0x43, 0x00 }, /* Ç to C */
- { 0x00C8, 0x45, 0x00 }, /* È to E */
- { 0x00C9, 0x45, 0x00 }, /* É to E */
- { 0x00CA, 0x45, 0x00 }, /* Ê to E */
- { 0x00CB, 0x45, 0x00 }, /* Ë to E */
- { 0x00CC, 0x49, 0x00 }, /* Ì to I */
- { 0x00CD, 0x49, 0x00 }, /* Í to I */
- { 0x00CE, 0x49, 0x00 }, /* Î to I */
- { 0x00CF, 0x49, 0x00 }, /* Ï to I */
- { 0x00D0, 0x44, 0x00 }, /* Ð to D */
- { 0x00D1, 0x4E, 0x00 }, /* Ñ to N */
- { 0x00D2, 0x4F, 0x00 }, /* Ò to O */
- { 0x00D3, 0x4F, 0x00 }, /* Ó to O */
- { 0x00D4, 0x4F, 0x00 }, /* Ô to O */
- { 0x00D5, 0x4F, 0x00 }, /* Õ to O */
- { 0x00D6, 0x4F, 0x65 }, /* Ö to Oe */
- { 0x00D7, 0x78, 0x00 }, /* × to x */
- { 0x00D8, 0x4F, 0x00 }, /* Ø to O */
- { 0x00D9, 0x55, 0x00 }, /* Ù to U */
- { 0x00DA, 0x55, 0x00 }, /* Ú to U */
- { 0x00DB, 0x55, 0x00 }, /* Û to U */
- { 0x00DC, 0x55, 0x65 }, /* Ü to Ue */
- { 0x00DD, 0x59, 0x00 }, /* Ý to Y */
- { 0x00DE, 0x54, 0x68 }, /* Þ to Th */
- { 0x00DF, 0x73, 0x73 }, /* ß to ss */
- { 0x00E0, 0x61, 0x00 }, /* à to a */
- { 0x00E1, 0x61, 0x00 }, /* á to a */
- { 0x00E2, 0x61, 0x00 }, /* â to a */
- { 0x00E3, 0x61, 0x00 }, /* ã to a */
- { 0x00E4, 0x61, 0x65 }, /* ä to ae */
- { 0x00E5, 0x61, 0x61 }, /* å to aa */
- { 0x00E6, 0x61, 0x65 }, /* æ to ae */
- { 0x00E7, 0x63, 0x00 }, /* ç to c */
- { 0x00E8, 0x65, 0x00 }, /* è to e */
- { 0x00E9, 0x65, 0x00 }, /* é to e */
- { 0x00EA, 0x65, 0x00 }, /* ê to e */
- { 0x00EB, 0x65, 0x00 }, /* ë to e */
- { 0x00EC, 0x69, 0x00 }, /* ì to i */
- { 0x00ED, 0x69, 0x00 }, /* í to i */
- { 0x00EE, 0x69, 0x00 }, /* î to i */
- { 0x00EF, 0x69, 0x00 }, /* ï to i */
- { 0x00F0, 0x64, 0x00 }, /* ð to d */
- { 0x00F1, 0x6E, 0x00 }, /* ñ to n */
- { 0x00F2, 0x6F, 0x00 }, /* ò to o */
- { 0x00F3, 0x6F, 0x00 }, /* ó to o */
- { 0x00F4, 0x6F, 0x00 }, /* ô to o */
- { 0x00F5, 0x6F, 0x00 }, /* õ to o */
- { 0x00F6, 0x6F, 0x65 }, /* ö to oe */
- { 0x00F7, 0x3A, 0x00 }, /* ÷ to : */
- { 0x00F8, 0x6F, 0x00 }, /* ø to o */
- { 0x00F9, 0x75, 0x00 }, /* ù to u */
- { 0x00FA, 0x75, 0x00 }, /* ú to u */
- { 0x00FB, 0x75, 0x00 }, /* û to u */
- { 0x00FC, 0x75, 0x65 }, /* ü to ue */
- { 0x00FD, 0x79, 0x00 }, /* ý to y */
- { 0x00FE, 0x74, 0x68 }, /* þ to th */
- { 0x00FF, 0x79, 0x00 }, /* ÿ to y */
- { 0x0100, 0x41, 0x00 }, /* Ā to A */
- { 0x0101, 0x61, 0x00 }, /* ā to a */
- { 0x0102, 0x41, 0x00 }, /* Ă to A */
- { 0x0103, 0x61, 0x00 }, /* ă to a */
- { 0x0104, 0x41, 0x00 }, /* Ą to A */
- { 0x0105, 0x61, 0x00 }, /* ą to a */
- { 0x0106, 0x43, 0x00 }, /* Ć to C */
- { 0x0107, 0x63, 0x00 }, /* ć to c */
- { 0x0108, 0x43, 0x68 }, /* Ĉ to Ch */
- { 0x0109, 0x63, 0x68 }, /* ĉ to ch */
- { 0x010A, 0x43, 0x00 }, /* Ċ to C */
- { 0x010B, 0x63, 0x00 }, /* ċ to c */
- { 0x010C, 0x43, 0x00 }, /* Č to C */
- { 0x010D, 0x63, 0x00 }, /* č to c */
- { 0x010E, 0x44, 0x00 }, /* Ď to D */
- { 0x010F, 0x64, 0x00 }, /* ď to d */
- { 0x0110, 0x44, 0x00 }, /* Đ to D */
- { 0x0111, 0x64, 0x00 }, /* đ to d */
- { 0x0112, 0x45, 0x00 }, /* Ē to E */
- { 0x0113, 0x65, 0x00 }, /* ē to e */
- { 0x0114, 0x45, 0x00 }, /* Ĕ to E */
- { 0x0115, 0x65, 0x00 }, /* ĕ to e */
- { 0x0116, 0x45, 0x00 }, /* Ė to E */
- { 0x0117, 0x65, 0x00 }, /* ė to e */
- { 0x0118, 0x45, 0x00 }, /* Ę to E */
- { 0x0119, 0x65, 0x00 }, /* ę to e */
- { 0x011A, 0x45, 0x00 }, /* Ě to E */
- { 0x011B, 0x65, 0x00 }, /* ě to e */
- { 0x011C, 0x47, 0x68 }, /* Ĝ to Gh */
- { 0x011D, 0x67, 0x68 }, /* ĝ to gh */
- { 0x011E, 0x47, 0x00 }, /* Ğ to G */
- { 0x011F, 0x67, 0x00 }, /* ğ to g */
- { 0x0120, 0x47, 0x00 }, /* Ġ to G */
- { 0x0121, 0x67, 0x00 }, /* ġ to g */
- { 0x0122, 0x47, 0x00 }, /* Ģ to G */
- { 0x0123, 0x67, 0x00 }, /* ģ to g */
- { 0x0124, 0x48, 0x68 }, /* Ĥ to Hh */
- { 0x0125, 0x68, 0x68 }, /* ĥ to hh */
- { 0x0126, 0x48, 0x00 }, /* Ħ to H */
- { 0x0127, 0x68, 0x00 }, /* ħ to h */
- { 0x0128, 0x49, 0x00 }, /* Ĩ to I */
- { 0x0129, 0x69, 0x00 }, /* ĩ to i */
- { 0x012A, 0x49, 0x00 }, /* Ī to I */
- { 0x012B, 0x69, 0x00 }, /* ī to i */
- { 0x012C, 0x49, 0x00 }, /* Ĭ to I */
- { 0x012D, 0x69, 0x00 }, /* ĭ to i */
- { 0x012E, 0x49, 0x00 }, /* Į to I */
- { 0x012F, 0x69, 0x00 }, /* į to i */
- { 0x0130, 0x49, 0x00 }, /* İ to I */
- { 0x0131, 0x69, 0x00 }, /* ı to i */
- { 0x0132, 0x49, 0x4A }, /* IJ to IJ */
- { 0x0133, 0x69, 0x6A }, /* ij to ij */
- { 0x0134, 0x4A, 0x68 }, /* Ĵ to Jh */
- { 0x0135, 0x6A, 0x68 }, /* ĵ to jh */
- { 0x0136, 0x4B, 0x00 }, /* Ķ to K */
- { 0x0137, 0x6B, 0x00 }, /* ķ to k */
- { 0x0138, 0x6B, 0x00 }, /* ĸ to k */
- { 0x0139, 0x4C, 0x00 }, /* Ĺ to L */
- { 0x013A, 0x6C, 0x00 }, /* ĺ to l */
- { 0x013B, 0x4C, 0x00 }, /* Ļ to L */
- { 0x013C, 0x6C, 0x00 }, /* ļ to l */
- { 0x013D, 0x4C, 0x00 }, /* Ľ to L */
- { 0x013E, 0x6C, 0x00 }, /* ľ to l */
- { 0x013F, 0x4C, 0x2E }, /* Ŀ to L. */
- { 0x0140, 0x6C, 0x2E }, /* ŀ to l. */
- { 0x0141, 0x4C, 0x00 }, /* Ł to L */
- { 0x0142, 0x6C, 0x00 }, /* ł to l */
- { 0x0143, 0x4E, 0x00 }, /* Ń to N */
- { 0x0144, 0x6E, 0x00 }, /* ń to n */
- { 0x0145, 0x4E, 0x00 }, /* Ņ to N */
- { 0x0146, 0x6E, 0x00 }, /* ņ to n */
- { 0x0147, 0x4E, 0x00 }, /* Ň to N */
- { 0x0148, 0x6E, 0x00 }, /* ň to n */
- { 0x0149, 0x27, 0x6E }, /* ʼn to 'n */
- { 0x014A, 0x4E, 0x47 }, /* Ŋ to NG */
- { 0x014B, 0x6E, 0x67 }, /* ŋ to ng */
- { 0x014C, 0x4F, 0x00 }, /* Ō to O */
- { 0x014D, 0x6F, 0x00 }, /* ō to o */
- { 0x014E, 0x4F, 0x00 }, /* Ŏ to O */
- { 0x014F, 0x6F, 0x00 }, /* ŏ to o */
- { 0x0150, 0x4F, 0x00 }, /* Ő to O */
- { 0x0151, 0x6F, 0x00 }, /* ő to o */
- { 0x0152, 0x4F, 0x45 }, /* Πto OE */
- { 0x0153, 0x6F, 0x65 }, /* œ to oe */
- { 0x0154, 0x52, 0x00 }, /* Ŕ to R */
- { 0x0155, 0x72, 0x00 }, /* ŕ to r */
- { 0x0156, 0x52, 0x00 }, /* Ŗ to R */
- { 0x0157, 0x72, 0x00 }, /* ŗ to r */
- { 0x0158, 0x52, 0x00 }, /* Ř to R */
- { 0x0159, 0x72, 0x00 }, /* ř to r */
- { 0x015A, 0x53, 0x00 }, /* Ś to S */
- { 0x015B, 0x73, 0x00 }, /* ś to s */
- { 0x015C, 0x53, 0x68 }, /* Ŝ to Sh */
- { 0x015D, 0x73, 0x68 }, /* ŝ to sh */
- { 0x015E, 0x53, 0x00 }, /* Ş to S */
- { 0x015F, 0x73, 0x00 }, /* ş to s */
- { 0x0160, 0x53, 0x00 }, /* Š to S */
- { 0x0161, 0x73, 0x00 }, /* š to s */
- { 0x0162, 0x54, 0x00 }, /* Ţ to T */
- { 0x0163, 0x74, 0x00 }, /* ţ to t */
- { 0x0164, 0x54, 0x00 }, /* Ť to T */
- { 0x0165, 0x74, 0x00 }, /* ť to t */
- { 0x0166, 0x54, 0x00 }, /* Ŧ to T */
- { 0x0167, 0x74, 0x00 }, /* ŧ to t */
- { 0x0168, 0x55, 0x00 }, /* Ũ to U */
- { 0x0169, 0x75, 0x00 }, /* ũ to u */
- { 0x016A, 0x55, 0x00 }, /* Ū to U */
- { 0x016B, 0x75, 0x00 }, /* ū to u */
- { 0x016C, 0x55, 0x00 }, /* Ŭ to U */
- { 0x016D, 0x75, 0x00 }, /* ŭ to u */
- { 0x016E, 0x55, 0x00 }, /* Ů to U */
- { 0x016F, 0x75, 0x00 }, /* ů to u */
- { 0x0170, 0x55, 0x00 }, /* Ű to U */
- { 0x0171, 0x75, 0x00 }, /* ű to u */
- { 0x0172, 0x55, 0x00 }, /* Ų to U */
- { 0x0173, 0x75, 0x00 }, /* ų to u */
- { 0x0174, 0x57, 0x00 }, /* Ŵ to W */
- { 0x0175, 0x77, 0x00 }, /* ŵ to w */
- { 0x0176, 0x59, 0x00 }, /* Ŷ to Y */
- { 0x0177, 0x79, 0x00 }, /* ŷ to y */
- { 0x0178, 0x59, 0x00 }, /* Ÿ to Y */
- { 0x0179, 0x5A, 0x00 }, /* Ź to Z */
- { 0x017A, 0x7A, 0x00 }, /* ź to z */
- { 0x017B, 0x5A, 0x00 }, /* Ż to Z */
- { 0x017C, 0x7A, 0x00 }, /* ż to z */
- { 0x017D, 0x5A, 0x00 }, /* Ž to Z */
- { 0x017E, 0x7A, 0x00 }, /* ž to z */
- { 0x017F, 0x73, 0x00 }, /* ſ to s */
- { 0x0192, 0x66, 0x00 }, /* ƒ to f */
- { 0x0218, 0x53, 0x00 }, /* Ș to S */
- { 0x0219, 0x73, 0x00 }, /* ș to s */
- { 0x021A, 0x54, 0x00 }, /* Ț to T */
- { 0x021B, 0x74, 0x00 }, /* ț to t */
- { 0x0386, 0x41, 0x00 }, /* Ά to A */
- { 0x0388, 0x45, 0x00 }, /* Έ to E */
- { 0x0389, 0x49, 0x00 }, /* Ή to I */
- { 0x038A, 0x49, 0x00 }, /* Ί to I */
- { 0x038C, 0x4f, 0x00 }, /* Ό to O */
- { 0x038E, 0x59, 0x00 }, /* Ύ to Y */
- { 0x038F, 0x4f, 0x00 }, /* Ώ to O */
- { 0x0390, 0x69, 0x00 }, /* ΐ to i */
- { 0x0391, 0x41, 0x00 }, /* Α to A */
- { 0x0392, 0x42, 0x00 }, /* Β to B */
- { 0x0393, 0x47, 0x00 }, /* Γ to G */
- { 0x0394, 0x44, 0x00 }, /* Δ to D */
- { 0x0395, 0x45, 0x00 }, /* Ε to E */
- { 0x0396, 0x5a, 0x00 }, /* Ζ to Z */
- { 0x0397, 0x49, 0x00 }, /* Η to I */
- { 0x0398, 0x54, 0x68 }, /* Θ to Th */
- { 0x0399, 0x49, 0x00 }, /* Ι to I */
- { 0x039A, 0x4b, 0x00 }, /* Κ to K */
- { 0x039B, 0x4c, 0x00 }, /* Λ to L */
- { 0x039C, 0x4d, 0x00 }, /* Μ to M */
- { 0x039D, 0x4e, 0x00 }, /* Ν to N */
- { 0x039E, 0x58, 0x00 }, /* Ξ to X */
- { 0x039F, 0x4f, 0x00 }, /* Ο to O */
- { 0x03A0, 0x50, 0x00 }, /* Π to P */
- { 0x03A1, 0x52, 0x00 }, /* Ρ to R */
- { 0x03A3, 0x53, 0x00 }, /* Σ to S */
- { 0x03A4, 0x54, 0x00 }, /* Τ to T */
- { 0x03A5, 0x59, 0x00 }, /* Υ to Y */
- { 0x03A6, 0x46, 0x00 }, /* Φ to F */
- { 0x03A7, 0x43, 0x68 }, /* Χ to Ch */
- { 0x03A8, 0x50, 0x73 }, /* Ψ to Ps */
- { 0x03A9, 0x4f, 0x00 }, /* Ω to O */
- { 0x03AA, 0x49, 0x00 }, /* Ϊ to I */
- { 0x03AB, 0x59, 0x00 }, /* Ϋ to Y */
- { 0x03AC, 0x61, 0x00 }, /* ά to a */
- { 0x03AD, 0x65, 0x00 }, /* έ to e */
- { 0x03AE, 0x69, 0x00 }, /* ή to i */
- { 0x03AF, 0x69, 0x00 }, /* ί to i */
- { 0x03B1, 0x61, 0x00 }, /* α to a */
- { 0x03B2, 0x62, 0x00 }, /* β to b */
- { 0x03B3, 0x67, 0x00 }, /* γ to g */
- { 0x03B4, 0x64, 0x00 }, /* δ to d */
- { 0x03B5, 0x65, 0x00 }, /* ε to e */
- { 0x03B6, 0x7a, 0x00 }, /* ζ to z */
- { 0x03B7, 0x69, 0x00 }, /* η to i */
- { 0x03B8, 0x74, 0x68 }, /* θ to th */
- { 0x03B9, 0x69, 0x00 }, /* ι to i */
- { 0x03BA, 0x6b, 0x00 }, /* κ to k */
- { 0x03BB, 0x6c, 0x00 }, /* λ to l */
- { 0x03BC, 0x6d, 0x00 }, /* μ to m */
- { 0x03BD, 0x6e, 0x00 }, /* ν to n */
- { 0x03BE, 0x78, 0x00 }, /* ξ to x */
- { 0x03BF, 0x6f, 0x00 }, /* ο to o */
- { 0x03C0, 0x70, 0x00 }, /* π to p */
- { 0x03C1, 0x72, 0x00 }, /* ρ to r */
- { 0x03C3, 0x73, 0x00 }, /* σ to s */
- { 0x03C4, 0x74, 0x00 }, /* τ to t */
- { 0x03C5, 0x79, 0x00 }, /* υ to y */
- { 0x03C6, 0x66, 0x00 }, /* φ to f */
- { 0x03C7, 0x63, 0x68 }, /* χ to ch */
- { 0x03C8, 0x70, 0x73 }, /* ψ to ps */
- { 0x03C9, 0x6f, 0x00 }, /* ω to o */
- { 0x03CA, 0x69, 0x00 }, /* ϊ to i */
- { 0x03CB, 0x79, 0x00 }, /* ϋ to y */
- { 0x03CC, 0x6f, 0x00 }, /* ό to o */
- { 0x03CD, 0x79, 0x00 }, /* ύ to y */
- { 0x03CE, 0x69, 0x00 }, /* ώ to i */
- { 0x0400, 0x45, 0x00 }, /* Ѐ to E */
- { 0x0401, 0x45, 0x00 }, /* Ё to E */
- { 0x0402, 0x44, 0x00 }, /* Ђ to D */
- { 0x0403, 0x47, 0x00 }, /* Ѓ to G */
- { 0x0404, 0x45, 0x00 }, /* Є to E */
- { 0x0405, 0x5a, 0x00 }, /* Ѕ to Z */
- { 0x0406, 0x49, 0x00 }, /* І to I */
- { 0x0407, 0x49, 0x00 }, /* Ї to I */
- { 0x0408, 0x4a, 0x00 }, /* Ј to J */
- { 0x0409, 0x49, 0x00 }, /* Љ to I */
- { 0x040A, 0x4e, 0x00 }, /* Њ to N */
- { 0x040B, 0x44, 0x00 }, /* Ћ to D */
- { 0x040C, 0x4b, 0x00 }, /* Ќ to K */
- { 0x040D, 0x49, 0x00 }, /* Ѝ to I */
- { 0x040E, 0x55, 0x00 }, /* Ў to U */
- { 0x040F, 0x44, 0x00 }, /* Џ to D */
- { 0x0410, 0x41, 0x00 }, /* А to A */
- { 0x0411, 0x42, 0x00 }, /* Б to B */
- { 0x0412, 0x56, 0x00 }, /* В to V */
- { 0x0413, 0x47, 0x00 }, /* Г to G */
- { 0x0414, 0x44, 0x00 }, /* Д to D */
- { 0x0415, 0x45, 0x00 }, /* Е to E */
- { 0x0416, 0x5a, 0x68 }, /* Ж to Zh */
- { 0x0417, 0x5a, 0x00 }, /* З to Z */
- { 0x0418, 0x49, 0x00 }, /* И to I */
- { 0x0419, 0x49, 0x00 }, /* Й to I */
- { 0x041A, 0x4b, 0x00 }, /* К to K */
- { 0x041B, 0x4c, 0x00 }, /* Л to L */
- { 0x041C, 0x4d, 0x00 }, /* М to M */
- { 0x041D, 0x4e, 0x00 }, /* Н to N */
- { 0x041E, 0x4f, 0x00 }, /* О to O */
- { 0x041F, 0x50, 0x00 }, /* П to P */
- { 0x0420, 0x52, 0x00 }, /* Р to R */
- { 0x0421, 0x53, 0x00 }, /* С to S */
- { 0x0422, 0x54, 0x00 }, /* Т to T */
- { 0x0423, 0x55, 0x00 }, /* У to U */
- { 0x0424, 0x46, 0x00 }, /* Ф to F */
- { 0x0425, 0x4b, 0x68 }, /* Х to Kh */
- { 0x0426, 0x54, 0x63 }, /* Ц to Tc */
- { 0x0427, 0x43, 0x68 }, /* Ч to Ch */
- { 0x0428, 0x53, 0x68 }, /* Ш to Sh */
- { 0x0429, 0x53, 0x68 }, /* Щ to Shch */
- { 0x042A, 0x61, 0x00 }, /* to A */
- { 0x042B, 0x59, 0x00 }, /* Ы to Y */
- { 0x042C, 0x59, 0x00 }, /* to Y */
- { 0x042D, 0x45, 0x00 }, /* Э to E */
- { 0x042E, 0x49, 0x75 }, /* Ю to Iu */
- { 0x042F, 0x49, 0x61 }, /* Я to Ia */
- { 0x0430, 0x61, 0x00 }, /* а to a */
- { 0x0431, 0x62, 0x00 }, /* б to b */
- { 0x0432, 0x76, 0x00 }, /* в to v */
- { 0x0433, 0x67, 0x00 }, /* г to g */
- { 0x0434, 0x64, 0x00 }, /* д to d */
- { 0x0435, 0x65, 0x00 }, /* е to e */
- { 0x0436, 0x7a, 0x68 }, /* ж to zh */
- { 0x0437, 0x7a, 0x00 }, /* з to z */
- { 0x0438, 0x69, 0x00 }, /* и to i */
- { 0x0439, 0x69, 0x00 }, /* й to i */
- { 0x043A, 0x6b, 0x00 }, /* к to k */
- { 0x043B, 0x6c, 0x00 }, /* л to l */
- { 0x043C, 0x6d, 0x00 }, /* м to m */
- { 0x043D, 0x6e, 0x00 }, /* н to n */
- { 0x043E, 0x6f, 0x00 }, /* о to o */
- { 0x043F, 0x70, 0x00 }, /* п to p */
- { 0x0440, 0x72, 0x00 }, /* р to r */
- { 0x0441, 0x73, 0x00 }, /* с to s */
- { 0x0442, 0x74, 0x00 }, /* т to t */
- { 0x0443, 0x75, 0x00 }, /* у to u */
- { 0x0444, 0x66, 0x00 }, /* ф to f */
- { 0x0445, 0x6b, 0x68 }, /* х to kh */
- { 0x0446, 0x74, 0x63 }, /* ц to tc */
- { 0x0447, 0x63, 0x68 }, /* ч to ch */
- { 0x0448, 0x73, 0x68 }, /* ш to sh */
- { 0x0449, 0x73, 0x68 }, /* щ to shch */
- { 0x044A, 0x61, 0x00 }, /* to a */
- { 0x044B, 0x79, 0x00 }, /* ы to y */
- { 0x044C, 0x79, 0x00 }, /* to y */
- { 0x044D, 0x65, 0x00 }, /* э to e */
- { 0x044E, 0x69, 0x75 }, /* ю to iu */
- { 0x044F, 0x69, 0x61 }, /* я to ia */
- { 0x0450, 0x65, 0x00 }, /* ѐ to e */
- { 0x0451, 0x65, 0x00 }, /* ё to e */
- { 0x0452, 0x64, 0x00 }, /* ђ to d */
- { 0x0453, 0x67, 0x00 }, /* ѓ to g */
- { 0x0454, 0x65, 0x00 }, /* є to e */
- { 0x0455, 0x7a, 0x00 }, /* ѕ to z */
- { 0x0456, 0x69, 0x00 }, /* і to i */
- { 0x0457, 0x69, 0x00 }, /* ї to i */
- { 0x0458, 0x6a, 0x00 }, /* ј to j */
- { 0x0459, 0x69, 0x00 }, /* љ to i */
- { 0x045A, 0x6e, 0x00 }, /* њ to n */
- { 0x045B, 0x64, 0x00 }, /* ћ to d */
- { 0x045C, 0x6b, 0x00 }, /* ќ to k */
- { 0x045D, 0x69, 0x00 }, /* ѝ to i */
- { 0x045E, 0x75, 0x00 }, /* ў to u */
- { 0x045F, 0x64, 0x00 }, /* џ to d */
- { 0x1E02, 0x42, 0x00 }, /* Ḃ to B */
- { 0x1E03, 0x62, 0x00 }, /* ḃ to b */
- { 0x1E0A, 0x44, 0x00 }, /* Ḋ to D */
- { 0x1E0B, 0x64, 0x00 }, /* ḋ to d */
- { 0x1E1E, 0x46, 0x00 }, /* Ḟ to F */
- { 0x1E1F, 0x66, 0x00 }, /* ḟ to f */
- { 0x1E40, 0x4D, 0x00 }, /* Ṁ to M */
- { 0x1E41, 0x6D, 0x00 }, /* ṁ to m */
- { 0x1E56, 0x50, 0x00 }, /* Ṗ to P */
- { 0x1E57, 0x70, 0x00 }, /* ṗ to p */
- { 0x1E60, 0x53, 0x00 }, /* Ṡ to S */
- { 0x1E61, 0x73, 0x00 }, /* ṡ to s */
- { 0x1E6A, 0x54, 0x00 }, /* Ṫ to T */
- { 0x1E6B, 0x74, 0x00 }, /* ṫ to t */
- { 0x1E80, 0x57, 0x00 }, /* Ẁ to W */
- { 0x1E81, 0x77, 0x00 }, /* ẁ to w */
- { 0x1E82, 0x57, 0x00 }, /* Ẃ to W */
- { 0x1E83, 0x77, 0x00 }, /* ẃ to w */
- { 0x1E84, 0x57, 0x00 }, /* Ẅ to W */
- { 0x1E85, 0x77, 0x00 }, /* ẅ to w */
- { 0x1EF2, 0x59, 0x00 }, /* Ỳ to Y */
- { 0x1EF3, 0x79, 0x00 }, /* ỳ to y */
- { 0xFB00, 0x66, 0x66 }, /* ff to ff */
- { 0xFB01, 0x66, 0x69 }, /* fi to fi */
- { 0xFB02, 0x66, 0x6C }, /* fl to fl */
- { 0xFB05, 0x73, 0x74 }, /* ſt to st */
- { 0xFB06, 0x73, 0x74 }, /* st to st */
+static const Transliteration translit[] = {
+ { 0x00A0, 0x20, 0x00, 0x00, 0x00 }, /* to */
+ { 0x00B5, 0x75, 0x00, 0x00, 0x00 }, /* µ to u */
+ { 0x00C0, 0x41, 0x00, 0x00, 0x00 }, /* À to A */
+ { 0x00C1, 0x41, 0x00, 0x00, 0x00 }, /* Á to A */
+ { 0x00C2, 0x41, 0x00, 0x00, 0x00 }, /* Â to A */
+ { 0x00C3, 0x41, 0x00, 0x00, 0x00 }, /* Ã to A */
+ { 0x00C4, 0x41, 0x65, 0x00, 0x00 }, /* Ä to Ae */
+ { 0x00C5, 0x41, 0x61, 0x00, 0x00 }, /* Å to Aa */
+ { 0x00C6, 0x41, 0x45, 0x00, 0x00 }, /* Æ to AE */
+ { 0x00C7, 0x43, 0x00, 0x00, 0x00 }, /* Ç to C */
+ { 0x00C8, 0x45, 0x00, 0x00, 0x00 }, /* È to E */
+ { 0x00C9, 0x45, 0x00, 0x00, 0x00 }, /* É to E */
+ { 0x00CA, 0x45, 0x00, 0x00, 0x00 }, /* Ê to E */
+ { 0x00CB, 0x45, 0x00, 0x00, 0x00 }, /* Ë to E */
+ { 0x00CC, 0x49, 0x00, 0x00, 0x00 }, /* Ì to I */
+ { 0x00CD, 0x49, 0x00, 0x00, 0x00 }, /* Í to I */
+ { 0x00CE, 0x49, 0x00, 0x00, 0x00 }, /* Î to I */
+ { 0x00CF, 0x49, 0x00, 0x00, 0x00 }, /* Ï to I */
+ { 0x00D0, 0x44, 0x00, 0x00, 0x00 }, /* Ð to D */
+ { 0x00D1, 0x4E, 0x00, 0x00, 0x00 }, /* Ñ to N */
+ { 0x00D2, 0x4F, 0x00, 0x00, 0x00 }, /* Ò to O */
+ { 0x00D3, 0x4F, 0x00, 0x00, 0x00 }, /* Ó to O */
+ { 0x00D4, 0x4F, 0x00, 0x00, 0x00 }, /* Ô to O */
+ { 0x00D5, 0x4F, 0x00, 0x00, 0x00 }, /* Õ to O */
+ { 0x00D6, 0x4F, 0x65, 0x00, 0x00 }, /* Ö to Oe */
+ { 0x00D7, 0x78, 0x00, 0x00, 0x00 }, /* × to x */
+ { 0x00D8, 0x4F, 0x00, 0x00, 0x00 }, /* Ø to O */
+ { 0x00D9, 0x55, 0x00, 0x00, 0x00 }, /* Ù to U */
+ { 0x00DA, 0x55, 0x00, 0x00, 0x00 }, /* Ú to U */
+ { 0x00DB, 0x55, 0x00, 0x00, 0x00 }, /* Û to U */
+ { 0x00DC, 0x55, 0x65, 0x00, 0x00 }, /* Ü to Ue */
+ { 0x00DD, 0x59, 0x00, 0x00, 0x00 }, /* Ý to Y */
+ { 0x00DE, 0x54, 0x68, 0x00, 0x00 }, /* Þ to Th */
+ { 0x00DF, 0x73, 0x73, 0x00, 0x00 }, /* ß to ss */
+ { 0x00E0, 0x61, 0x00, 0x00, 0x00 }, /* à to a */
+ { 0x00E1, 0x61, 0x00, 0x00, 0x00 }, /* á to a */
+ { 0x00E2, 0x61, 0x00, 0x00, 0x00 }, /* â to a */
+ { 0x00E3, 0x61, 0x00, 0x00, 0x00 }, /* ã to a */
+ { 0x00E4, 0x61, 0x65, 0x00, 0x00 }, /* ä to ae */
+ { 0x00E5, 0x61, 0x61, 0x00, 0x00 }, /* å to aa */
+ { 0x00E6, 0x61, 0x65, 0x00, 0x00 }, /* æ to ae */
+ { 0x00E7, 0x63, 0x00, 0x00, 0x00 }, /* ç to c */
+ { 0x00E8, 0x65, 0x00, 0x00, 0x00 }, /* è to e */
+ { 0x00E9, 0x65, 0x00, 0x00, 0x00 }, /* é to e */
+ { 0x00EA, 0x65, 0x00, 0x00, 0x00 }, /* ê to e */
+ { 0x00EB, 0x65, 0x00, 0x00, 0x00 }, /* ë to e */
+ { 0x00EC, 0x69, 0x00, 0x00, 0x00 }, /* ì to i */
+ { 0x00ED, 0x69, 0x00, 0x00, 0x00 }, /* í to i */
+ { 0x00EE, 0x69, 0x00, 0x00, 0x00 }, /* î to i */
+ { 0x00EF, 0x69, 0x00, 0x00, 0x00 }, /* ï to i */
+ { 0x00F0, 0x64, 0x00, 0x00, 0x00 }, /* ð to d */
+ { 0x00F1, 0x6E, 0x00, 0x00, 0x00 }, /* ñ to n */
+ { 0x00F2, 0x6F, 0x00, 0x00, 0x00 }, /* ò to o */
+ { 0x00F3, 0x6F, 0x00, 0x00, 0x00 }, /* ó to o */
+ { 0x00F4, 0x6F, 0x00, 0x00, 0x00 }, /* ô to o */
+ { 0x00F5, 0x6F, 0x00, 0x00, 0x00 }, /* õ to o */
+ { 0x00F6, 0x6F, 0x65, 0x00, 0x00 }, /* ö to oe */
+ { 0x00F7, 0x3A, 0x00, 0x00, 0x00 }, /* ÷ to : */
+ { 0x00F8, 0x6F, 0x00, 0x00, 0x00 }, /* ø to o */
+ { 0x00F9, 0x75, 0x00, 0x00, 0x00 }, /* ù to u */
+ { 0x00FA, 0x75, 0x00, 0x00, 0x00 }, /* ú to u */
+ { 0x00FB, 0x75, 0x00, 0x00, 0x00 }, /* û to u */
+ { 0x00FC, 0x75, 0x65, 0x00, 0x00 }, /* ü to ue */
+ { 0x00FD, 0x79, 0x00, 0x00, 0x00 }, /* ý to y */
+ { 0x00FE, 0x74, 0x68, 0x00, 0x00 }, /* þ to th */
+ { 0x00FF, 0x79, 0x00, 0x00, 0x00 }, /* ÿ to y */
+ { 0x0100, 0x41, 0x00, 0x00, 0x00 }, /* Ā to A */
+ { 0x0101, 0x61, 0x00, 0x00, 0x00 }, /* ā to a */
+ { 0x0102, 0x41, 0x00, 0x00, 0x00 }, /* Ă to A */
+ { 0x0103, 0x61, 0x00, 0x00, 0x00 }, /* ă to a */
+ { 0x0104, 0x41, 0x00, 0x00, 0x00 }, /* Ą to A */
+ { 0x0105, 0x61, 0x00, 0x00, 0x00 }, /* ą to a */
+ { 0x0106, 0x43, 0x00, 0x00, 0x00 }, /* Ć to C */
+ { 0x0107, 0x63, 0x00, 0x00, 0x00 }, /* ć to c */
+ { 0x0108, 0x43, 0x68, 0x00, 0x00 }, /* Ĉ to Ch */
+ { 0x0109, 0x63, 0x68, 0x00, 0x00 }, /* ĉ to ch */
+ { 0x010A, 0x43, 0x00, 0x00, 0x00 }, /* Ċ to C */
+ { 0x010B, 0x63, 0x00, 0x00, 0x00 }, /* ċ to c */
+ { 0x010C, 0x43, 0x00, 0x00, 0x00 }, /* Č to C */
+ { 0x010D, 0x63, 0x00, 0x00, 0x00 }, /* č to c */
+ { 0x010E, 0x44, 0x00, 0x00, 0x00 }, /* Ď to D */
+ { 0x010F, 0x64, 0x00, 0x00, 0x00 }, /* ď to d */
+ { 0x0110, 0x44, 0x00, 0x00, 0x00 }, /* Đ to D */
+ { 0x0111, 0x64, 0x00, 0x00, 0x00 }, /* đ to d */
+ { 0x0112, 0x45, 0x00, 0x00, 0x00 }, /* Ē to E */
+ { 0x0113, 0x65, 0x00, 0x00, 0x00 }, /* ē to e */
+ { 0x0114, 0x45, 0x00, 0x00, 0x00 }, /* Ĕ to E */
+ { 0x0115, 0x65, 0x00, 0x00, 0x00 }, /* ĕ to e */
+ { 0x0116, 0x45, 0x00, 0x00, 0x00 }, /* Ė to E */
+ { 0x0117, 0x65, 0x00, 0x00, 0x00 }, /* ė to e */
+ { 0x0118, 0x45, 0x00, 0x00, 0x00 }, /* Ę to E */
+ { 0x0119, 0x65, 0x00, 0x00, 0x00 }, /* ę to e */
+ { 0x011A, 0x45, 0x00, 0x00, 0x00 }, /* Ě to E */
+ { 0x011B, 0x65, 0x00, 0x00, 0x00 }, /* ě to e */
+ { 0x011C, 0x47, 0x68, 0x00, 0x00 }, /* Ĝ to Gh */
+ { 0x011D, 0x67, 0x68, 0x00, 0x00 }, /* ĝ to gh */
+ { 0x011E, 0x47, 0x00, 0x00, 0x00 }, /* Ğ to G */
+ { 0x011F, 0x67, 0x00, 0x00, 0x00 }, /* ğ to g */
+ { 0x0120, 0x47, 0x00, 0x00, 0x00 }, /* Ġ to G */
+ { 0x0121, 0x67, 0x00, 0x00, 0x00 }, /* ġ to g */
+ { 0x0122, 0x47, 0x00, 0x00, 0x00 }, /* Ģ to G */
+ { 0x0123, 0x67, 0x00, 0x00, 0x00 }, /* ģ to g */
+ { 0x0124, 0x48, 0x68, 0x00, 0x00 }, /* Ĥ to Hh */
+ { 0x0125, 0x68, 0x68, 0x00, 0x00 }, /* ĥ to hh */
+ { 0x0126, 0x48, 0x00, 0x00, 0x00 }, /* Ħ to H */
+ { 0x0127, 0x68, 0x00, 0x00, 0x00 }, /* ħ to h */
+ { 0x0128, 0x49, 0x00, 0x00, 0x00 }, /* Ĩ to I */
+ { 0x0129, 0x69, 0x00, 0x00, 0x00 }, /* ĩ to i */
+ { 0x012A, 0x49, 0x00, 0x00, 0x00 }, /* Ī to I */
+ { 0x012B, 0x69, 0x00, 0x00, 0x00 }, /* ī to i */
+ { 0x012C, 0x49, 0x00, 0x00, 0x00 }, /* Ĭ to I */
+ { 0x012D, 0x69, 0x00, 0x00, 0x00 }, /* ĭ to i */
+ { 0x012E, 0x49, 0x00, 0x00, 0x00 }, /* Į to I */
+ { 0x012F, 0x69, 0x00, 0x00, 0x00 }, /* į to i */
+ { 0x0130, 0x49, 0x00, 0x00, 0x00 }, /* İ to I */
+ { 0x0131, 0x69, 0x00, 0x00, 0x00 }, /* ı to i */
+ { 0x0132, 0x49, 0x4A, 0x00, 0x00 }, /* IJ to IJ */
+ { 0x0133, 0x69, 0x6A, 0x00, 0x00 }, /* ij to ij */
+ { 0x0134, 0x4A, 0x68, 0x00, 0x00 }, /* Ĵ to Jh */
+ { 0x0135, 0x6A, 0x68, 0x00, 0x00 }, /* ĵ to jh */
+ { 0x0136, 0x4B, 0x00, 0x00, 0x00 }, /* Ķ to K */
+ { 0x0137, 0x6B, 0x00, 0x00, 0x00 }, /* ķ to k */
+ { 0x0138, 0x6B, 0x00, 0x00, 0x00 }, /* ĸ to k */
+ { 0x0139, 0x4C, 0x00, 0x00, 0x00 }, /* Ĺ to L */
+ { 0x013A, 0x6C, 0x00, 0x00, 0x00 }, /* ĺ to l */
+ { 0x013B, 0x4C, 0x00, 0x00, 0x00 }, /* Ļ to L */
+ { 0x013C, 0x6C, 0x00, 0x00, 0x00 }, /* ļ to l */
+ { 0x013D, 0x4C, 0x00, 0x00, 0x00 }, /* Ľ to L */
+ { 0x013E, 0x6C, 0x00, 0x00, 0x00 }, /* ľ to l */
+ { 0x013F, 0x4C, 0x2E, 0x00, 0x00 }, /* Ŀ to L. */
+ { 0x0140, 0x6C, 0x2E, 0x00, 0x00 }, /* ŀ to l. */
+ { 0x0141, 0x4C, 0x00, 0x00, 0x00 }, /* Ł to L */
+ { 0x0142, 0x6C, 0x00, 0x00, 0x00 }, /* ł to l */
+ { 0x0143, 0x4E, 0x00, 0x00, 0x00 }, /* Ń to N */
+ { 0x0144, 0x6E, 0x00, 0x00, 0x00 }, /* ń to n */
+ { 0x0145, 0x4E, 0x00, 0x00, 0x00 }, /* Ņ to N */
+ { 0x0146, 0x6E, 0x00, 0x00, 0x00 }, /* ņ to n */
+ { 0x0147, 0x4E, 0x00, 0x00, 0x00 }, /* Ň to N */
+ { 0x0148, 0x6E, 0x00, 0x00, 0x00 }, /* ň to n */
+ { 0x0149, 0x27, 0x6E, 0x00, 0x00 }, /* ʼn to 'n */
+ { 0x014A, 0x4E, 0x47, 0x00, 0x00 }, /* Ŋ to NG */
+ { 0x014B, 0x6E, 0x67, 0x00, 0x00 }, /* ŋ to ng */
+ { 0x014C, 0x4F, 0x00, 0x00, 0x00 }, /* Ō to O */
+ { 0x014D, 0x6F, 0x00, 0x00, 0x00 }, /* ō to o */
+ { 0x014E, 0x4F, 0x00, 0x00, 0x00 }, /* Ŏ to O */
+ { 0x014F, 0x6F, 0x00, 0x00, 0x00 }, /* ŏ to o */
+ { 0x0150, 0x4F, 0x00, 0x00, 0x00 }, /* Ő to O */
+ { 0x0151, 0x6F, 0x00, 0x00, 0x00 }, /* ő to o */
+ { 0x0152, 0x4F, 0x45, 0x00, 0x00 }, /* Πto OE */
+ { 0x0153, 0x6F, 0x65, 0x00, 0x00 }, /* œ to oe */
+ { 0x0154, 0x52, 0x00, 0x00, 0x00 }, /* Ŕ to R */
+ { 0x0155, 0x72, 0x00, 0x00, 0x00 }, /* ŕ to r */
+ { 0x0156, 0x52, 0x00, 0x00, 0x00 }, /* Ŗ to R */
+ { 0x0157, 0x72, 0x00, 0x00, 0x00 }, /* ŗ to r */
+ { 0x0158, 0x52, 0x00, 0x00, 0x00 }, /* Ř to R */
+ { 0x0159, 0x72, 0x00, 0x00, 0x00 }, /* ř to r */
+ { 0x015A, 0x53, 0x00, 0x00, 0x00 }, /* Ś to S */
+ { 0x015B, 0x73, 0x00, 0x00, 0x00 }, /* ś to s */
+ { 0x015C, 0x53, 0x68, 0x00, 0x00 }, /* Ŝ to Sh */
+ { 0x015D, 0x73, 0x68, 0x00, 0x00 }, /* ŝ to sh */
+ { 0x015E, 0x53, 0x00, 0x00, 0x00 }, /* Ş to S */
+ { 0x015F, 0x73, 0x00, 0x00, 0x00 }, /* ş to s */
+ { 0x0160, 0x53, 0x00, 0x00, 0x00 }, /* Š to S */
+ { 0x0161, 0x73, 0x00, 0x00, 0x00 }, /* š to s */
+ { 0x0162, 0x54, 0x00, 0x00, 0x00 }, /* Ţ to T */
+ { 0x0163, 0x74, 0x00, 0x00, 0x00 }, /* ţ to t */
+ { 0x0164, 0x54, 0x00, 0x00, 0x00 }, /* Ť to T */
+ { 0x0165, 0x74, 0x00, 0x00, 0x00 }, /* ť to t */
+ { 0x0166, 0x54, 0x00, 0x00, 0x00 }, /* Ŧ to T */
+ { 0x0167, 0x74, 0x00, 0x00, 0x00 }, /* ŧ to t */
+ { 0x0168, 0x55, 0x00, 0x00, 0x00 }, /* Ũ to U */
+ { 0x0169, 0x75, 0x00, 0x00, 0x00 }, /* ũ to u */
+ { 0x016A, 0x55, 0x00, 0x00, 0x00 }, /* Ū to U */
+ { 0x016B, 0x75, 0x00, 0x00, 0x00 }, /* ū to u */
+ { 0x016C, 0x55, 0x00, 0x00, 0x00 }, /* Ŭ to U */
+ { 0x016D, 0x75, 0x00, 0x00, 0x00 }, /* ŭ to u */
+ { 0x016E, 0x55, 0x00, 0x00, 0x00 }, /* Ů to U */
+ { 0x016F, 0x75, 0x00, 0x00, 0x00 }, /* ů to u */
+ { 0x0170, 0x55, 0x00, 0x00, 0x00 }, /* Ű to U */
+ { 0x0171, 0x75, 0x00, 0x00, 0x00 }, /* ű to u */
+ { 0x0172, 0x55, 0x00, 0x00, 0x00 }, /* Ų to U */
+ { 0x0173, 0x75, 0x00, 0x00, 0x00 }, /* ų to u */
+ { 0x0174, 0x57, 0x00, 0x00, 0x00 }, /* Ŵ to W */
+ { 0x0175, 0x77, 0x00, 0x00, 0x00 }, /* ŵ to w */
+ { 0x0176, 0x59, 0x00, 0x00, 0x00 }, /* Ŷ to Y */
+ { 0x0177, 0x79, 0x00, 0x00, 0x00 }, /* ŷ to y */
+ { 0x0178, 0x59, 0x00, 0x00, 0x00 }, /* Ÿ to Y */
+ { 0x0179, 0x5A, 0x00, 0x00, 0x00 }, /* Ź to Z */
+ { 0x017A, 0x7A, 0x00, 0x00, 0x00 }, /* ź to z */
+ { 0x017B, 0x5A, 0x00, 0x00, 0x00 }, /* Ż to Z */
+ { 0x017C, 0x7A, 0x00, 0x00, 0x00 }, /* ż to z */
+ { 0x017D, 0x5A, 0x00, 0x00, 0x00 }, /* Ž to Z */
+ { 0x017E, 0x7A, 0x00, 0x00, 0x00 }, /* ž to z */
+ { 0x017F, 0x73, 0x00, 0x00, 0x00 }, /* ſ to s */
+ { 0x0192, 0x66, 0x00, 0x00, 0x00 }, /* ƒ to f */
+ { 0x0218, 0x53, 0x00, 0x00, 0x00 }, /* Ș to S */
+ { 0x0219, 0x73, 0x00, 0x00, 0x00 }, /* ș to s */
+ { 0x021A, 0x54, 0x00, 0x00, 0x00 }, /* Ț to T */
+ { 0x021B, 0x74, 0x00, 0x00, 0x00 }, /* ț to t */
+ { 0x0386, 0x41, 0x00, 0x00, 0x00 }, /* Ά to A */
+ { 0x0388, 0x45, 0x00, 0x00, 0x00 }, /* Έ to E */
+ { 0x0389, 0x49, 0x00, 0x00, 0x00 }, /* Ή to I */
+ { 0x038A, 0x49, 0x00, 0x00, 0x00 }, /* Ί to I */
+ { 0x038C, 0x4f, 0x00, 0x00, 0x00 }, /* Ό to O */
+ { 0x038E, 0x59, 0x00, 0x00, 0x00 }, /* Ύ to Y */
+ { 0x038F, 0x4f, 0x00, 0x00, 0x00 }, /* Ώ to O */
+ { 0x0390, 0x69, 0x00, 0x00, 0x00 }, /* ΐ to i */
+ { 0x0391, 0x41, 0x00, 0x00, 0x00 }, /* Α to A */
+ { 0x0392, 0x42, 0x00, 0x00, 0x00 }, /* Β to B */
+ { 0x0393, 0x47, 0x00, 0x00, 0x00 }, /* Γ to G */
+ { 0x0394, 0x44, 0x00, 0x00, 0x00 }, /* Δ to D */
+ { 0x0395, 0x45, 0x00, 0x00, 0x00 }, /* Ε to E */
+ { 0x0396, 0x5a, 0x00, 0x00, 0x00 }, /* Ζ to Z */
+ { 0x0397, 0x49, 0x00, 0x00, 0x00 }, /* Η to I */
+ { 0x0398, 0x54, 0x68, 0x00, 0x00 }, /* Θ to Th */
+ { 0x0399, 0x49, 0x00, 0x00, 0x00 }, /* Ι to I */
+ { 0x039A, 0x4b, 0x00, 0x00, 0x00 }, /* Κ to K */
+ { 0x039B, 0x4c, 0x00, 0x00, 0x00 }, /* Λ to L */
+ { 0x039C, 0x4d, 0x00, 0x00, 0x00 }, /* Μ to M */
+ { 0x039D, 0x4e, 0x00, 0x00, 0x00 }, /* Ν to N */
+ { 0x039E, 0x58, 0x00, 0x00, 0x00 }, /* Ξ to X */
+ { 0x039F, 0x4f, 0x00, 0x00, 0x00 }, /* Ο to O */
+ { 0x03A0, 0x50, 0x00, 0x00, 0x00 }, /* Π to P */
+ { 0x03A1, 0x52, 0x00, 0x00, 0x00 }, /* Ρ to R */
+ { 0x03A3, 0x53, 0x00, 0x00, 0x00 }, /* Σ to S */
+ { 0x03A4, 0x54, 0x00, 0x00, 0x00 }, /* Τ to T */
+ { 0x03A5, 0x59, 0x00, 0x00, 0x00 }, /* Υ to Y */
+ { 0x03A6, 0x46, 0x00, 0x00, 0x00 }, /* Φ to F */
+ { 0x03A7, 0x43, 0x68, 0x00, 0x00 }, /* Χ to Ch */
+ { 0x03A8, 0x50, 0x73, 0x00, 0x00 }, /* Ψ to Ps */
+ { 0x03A9, 0x4f, 0x00, 0x00, 0x00 }, /* Ω to O */
+ { 0x03AA, 0x49, 0x00, 0x00, 0x00 }, /* Ϊ to I */
+ { 0x03AB, 0x59, 0x00, 0x00, 0x00 }, /* Ϋ to Y */
+ { 0x03AC, 0x61, 0x00, 0x00, 0x00 }, /* ά to a */
+ { 0x03AD, 0x65, 0x00, 0x00, 0x00 }, /* έ to e */
+ { 0x03AE, 0x69, 0x00, 0x00, 0x00 }, /* ή to i */
+ { 0x03AF, 0x69, 0x00, 0x00, 0x00 }, /* ί to i */
+ { 0x03B1, 0x61, 0x00, 0x00, 0x00 }, /* α to a */
+ { 0x03B2, 0x62, 0x00, 0x00, 0x00 }, /* β to b */
+ { 0x03B3, 0x67, 0x00, 0x00, 0x00 }, /* γ to g */
+ { 0x03B4, 0x64, 0x00, 0x00, 0x00 }, /* δ to d */
+ { 0x03B5, 0x65, 0x00, 0x00, 0x00 }, /* ε to e */
+ { 0x03B6, 0x7a, 0x00, 0x00, 0x00 }, /* ζ to z */
+ { 0x03B7, 0x69, 0x00, 0x00, 0x00 }, /* η to i */
+ { 0x03B8, 0x74, 0x68, 0x00, 0x00 }, /* θ to th */
+ { 0x03B9, 0x69, 0x00, 0x00, 0x00 }, /* ι to i */
+ { 0x03BA, 0x6b, 0x00, 0x00, 0x00 }, /* κ to k */
+ { 0x03BB, 0x6c, 0x00, 0x00, 0x00 }, /* λ to l */
+ { 0x03BC, 0x6d, 0x00, 0x00, 0x00 }, /* μ to m */
+ { 0x03BD, 0x6e, 0x00, 0x00, 0x00 }, /* ν to n */
+ { 0x03BE, 0x78, 0x00, 0x00, 0x00 }, /* ξ to x */
+ { 0x03BF, 0x6f, 0x00, 0x00, 0x00 }, /* ο to o */
+ { 0x03C0, 0x70, 0x00, 0x00, 0x00 }, /* π to p */
+ { 0x03C1, 0x72, 0x00, 0x00, 0x00 }, /* ρ to r */
+ { 0x03C3, 0x73, 0x00, 0x00, 0x00 }, /* σ to s */
+ { 0x03C4, 0x74, 0x00, 0x00, 0x00 }, /* τ to t */
+ { 0x03C5, 0x79, 0x00, 0x00, 0x00 }, /* υ to y */
+ { 0x03C6, 0x66, 0x00, 0x00, 0x00 }, /* φ to f */
+ { 0x03C7, 0x63, 0x68, 0x00, 0x00 }, /* χ to ch */
+ { 0x03C8, 0x70, 0x73, 0x00, 0x00 }, /* ψ to ps */
+ { 0x03C9, 0x6f, 0x00, 0x00, 0x00 }, /* ω to o */
+ { 0x03CA, 0x69, 0x00, 0x00, 0x00 }, /* ϊ to i */
+ { 0x03CB, 0x79, 0x00, 0x00, 0x00 }, /* ϋ to y */
+ { 0x03CC, 0x6f, 0x00, 0x00, 0x00 }, /* ό to o */
+ { 0x03CD, 0x79, 0x00, 0x00, 0x00 }, /* ύ to y */
+ { 0x03CE, 0x69, 0x00, 0x00, 0x00 }, /* ώ to i */
+ { 0x0400, 0x45, 0x00, 0x00, 0x00 }, /* Ѐ to E */
+ { 0x0401, 0x45, 0x00, 0x00, 0x00 }, /* Ё to E */
+ { 0x0402, 0x44, 0x00, 0x00, 0x00 }, /* Ђ to D */
+ { 0x0403, 0x47, 0x00, 0x00, 0x00 }, /* Ѓ to G */
+ { 0x0404, 0x45, 0x00, 0x00, 0x00 }, /* Є to E */
+ { 0x0405, 0x5a, 0x00, 0x00, 0x00 }, /* Ѕ to Z */
+ { 0x0406, 0x49, 0x00, 0x00, 0x00 }, /* І to I */
+ { 0x0407, 0x49, 0x00, 0x00, 0x00 }, /* Ї to I */
+ { 0x0408, 0x4a, 0x00, 0x00, 0x00 }, /* Ј to J */
+ { 0x0409, 0x49, 0x00, 0x00, 0x00 }, /* Љ to I */
+ { 0x040A, 0x4e, 0x00, 0x00, 0x00 }, /* Њ to N */
+ { 0x040B, 0x44, 0x00, 0x00, 0x00 }, /* Ћ to D */
+ { 0x040C, 0x4b, 0x00, 0x00, 0x00 }, /* Ќ to K */
+ { 0x040D, 0x49, 0x00, 0x00, 0x00 }, /* Ѝ to I */
+ { 0x040E, 0x55, 0x00, 0x00, 0x00 }, /* Ў to U */
+ { 0x040F, 0x44, 0x00, 0x00, 0x00 }, /* Џ to D */
+ { 0x0410, 0x41, 0x00, 0x00, 0x00 }, /* А to A */
+ { 0x0411, 0x42, 0x00, 0x00, 0x00 }, /* Б to B */
+ { 0x0412, 0x56, 0x00, 0x00, 0x00 }, /* В to V */
+ { 0x0413, 0x47, 0x00, 0x00, 0x00 }, /* Г to G */
+ { 0x0414, 0x44, 0x00, 0x00, 0x00 }, /* Д to D */
+ { 0x0415, 0x45, 0x00, 0x00, 0x00 }, /* Е to E */
+ { 0x0416, 0x5a, 0x68, 0x00, 0x00 }, /* Ж to Zh */
+ { 0x0417, 0x5a, 0x00, 0x00, 0x00 }, /* З to Z */
+ { 0x0418, 0x49, 0x00, 0x00, 0x00 }, /* И to I */
+ { 0x0419, 0x49, 0x00, 0x00, 0x00 }, /* Й to I */
+ { 0x041A, 0x4b, 0x00, 0x00, 0x00 }, /* К to K */
+ { 0x041B, 0x4c, 0x00, 0x00, 0x00 }, /* Л to L */
+ { 0x041C, 0x4d, 0x00, 0x00, 0x00 }, /* М to M */
+ { 0x041D, 0x4e, 0x00, 0x00, 0x00 }, /* Н to N */
+ { 0x041E, 0x4f, 0x00, 0x00, 0x00 }, /* О to O */
+ { 0x041F, 0x50, 0x00, 0x00, 0x00 }, /* П to P */
+ { 0x0420, 0x52, 0x00, 0x00, 0x00 }, /* Р to R */
+ { 0x0421, 0x53, 0x00, 0x00, 0x00 }, /* С to S */
+ { 0x0422, 0x54, 0x00, 0x00, 0x00 }, /* Т to T */
+ { 0x0423, 0x55, 0x00, 0x00, 0x00 }, /* У to U */
+ { 0x0424, 0x46, 0x00, 0x00, 0x00 }, /* Ф to F */
+ { 0x0425, 0x4b, 0x68, 0x00, 0x00 }, /* Х to Kh */
+ { 0x0426, 0x54, 0x63, 0x00, 0x00 }, /* Ц to Tc */
+ { 0x0427, 0x43, 0x68, 0x00, 0x00 }, /* Ч to Ch */
+ { 0x0428, 0x53, 0x68, 0x00, 0x00 }, /* Ш to Sh */
+ { 0x0429, 0x53, 0x68, 0x63, 0x68 }, /* Щ to Shch */
+ { 0x042A, 0x61, 0x00, 0x00, 0x00 }, /* to A */
+ { 0x042B, 0x59, 0x00, 0x00, 0x00 }, /* Ы to Y */
+ { 0x042C, 0x59, 0x00, 0x00, 0x00 }, /* to Y */
+ { 0x042D, 0x45, 0x00, 0x00, 0x00 }, /* Э to E */
+ { 0x042E, 0x49, 0x75, 0x00, 0x00 }, /* Ю to Iu */
+ { 0x042F, 0x49, 0x61, 0x00, 0x00 }, /* Я to Ia */
+ { 0x0430, 0x61, 0x00, 0x00, 0x00 }, /* а to a */
+ { 0x0431, 0x62, 0x00, 0x00, 0x00 }, /* б to b */
+ { 0x0432, 0x76, 0x00, 0x00, 0x00 }, /* в to v */
+ { 0x0433, 0x67, 0x00, 0x00, 0x00 }, /* г to g */
+ { 0x0434, 0x64, 0x00, 0x00, 0x00 }, /* д to d */
+ { 0x0435, 0x65, 0x00, 0x00, 0x00 }, /* е to e */
+ { 0x0436, 0x7a, 0x68, 0x00, 0x00 }, /* ж to zh */
+ { 0x0437, 0x7a, 0x00, 0x00, 0x00 }, /* з to z */
+ { 0x0438, 0x69, 0x00, 0x00, 0x00 }, /* и to i */
+ { 0x0439, 0x69, 0x00, 0x00, 0x00 }, /* й to i */
+ { 0x043A, 0x6b, 0x00, 0x00, 0x00 }, /* к to k */
+ { 0x043B, 0x6c, 0x00, 0x00, 0x00 }, /* л to l */
+ { 0x043C, 0x6d, 0x00, 0x00, 0x00 }, /* м to m */
+ { 0x043D, 0x6e, 0x00, 0x00, 0x00 }, /* н to n */
+ { 0x043E, 0x6f, 0x00, 0x00, 0x00 }, /* о to o */
+ { 0x043F, 0x70, 0x00, 0x00, 0x00 }, /* п to p */
+ { 0x0440, 0x72, 0x00, 0x00, 0x00 }, /* р to r */
+ { 0x0441, 0x73, 0x00, 0x00, 0x00 }, /* с to s */
+ { 0x0442, 0x74, 0x00, 0x00, 0x00 }, /* т to t */
+ { 0x0443, 0x75, 0x00, 0x00, 0x00 }, /* у to u */
+ { 0x0444, 0x66, 0x00, 0x00, 0x00 }, /* ф to f */
+ { 0x0445, 0x6b, 0x68, 0x00, 0x00 }, /* х to kh */
+ { 0x0446, 0x74, 0x63, 0x00, 0x00 }, /* ц to tc */
+ { 0x0447, 0x63, 0x68, 0x00, 0x00 }, /* ч to ch */
+ { 0x0448, 0x73, 0x68, 0x00, 0x00 }, /* ш to sh */
+ { 0x0449, 0x73, 0x68, 0x63, 0x68 }, /* щ to shch */
+ { 0x044A, 0x61, 0x00, 0x00, 0x00 }, /* to a */
+ { 0x044B, 0x79, 0x00, 0x00, 0x00 }, /* ы to y */
+ { 0x044C, 0x79, 0x00, 0x00, 0x00 }, /* to y */
+ { 0x044D, 0x65, 0x00, 0x00, 0x00 }, /* э to e */
+ { 0x044E, 0x69, 0x75, 0x00, 0x00 }, /* ю to iu */
+ { 0x044F, 0x69, 0x61, 0x00, 0x00 }, /* я to ia */
+ { 0x0450, 0x65, 0x00, 0x00, 0x00 }, /* ѐ to e */
+ { 0x0451, 0x65, 0x00, 0x00, 0x00 }, /* ё to e */
+ { 0x0452, 0x64, 0x00, 0x00, 0x00 }, /* ђ to d */
+ { 0x0453, 0x67, 0x00, 0x00, 0x00 }, /* ѓ to g */
+ { 0x0454, 0x65, 0x00, 0x00, 0x00 }, /* є to e */
+ { 0x0455, 0x7a, 0x00, 0x00, 0x00 }, /* ѕ to z */
+ { 0x0456, 0x69, 0x00, 0x00, 0x00 }, /* і to i */
+ { 0x0457, 0x69, 0x00, 0x00, 0x00 }, /* ї to i */
+ { 0x0458, 0x6a, 0x00, 0x00, 0x00 }, /* ј to j */
+ { 0x0459, 0x69, 0x00, 0x00, 0x00 }, /* љ to i */
+ { 0x045A, 0x6e, 0x00, 0x00, 0x00 }, /* њ to n */
+ { 0x045B, 0x64, 0x00, 0x00, 0x00 }, /* ћ to d */
+ { 0x045C, 0x6b, 0x00, 0x00, 0x00 }, /* ќ to k */
+ { 0x045D, 0x69, 0x00, 0x00, 0x00 }, /* ѝ to i */
+ { 0x045E, 0x75, 0x00, 0x00, 0x00 }, /* ў to u */
+ { 0x045F, 0x64, 0x00, 0x00, 0x00 }, /* џ to d */
+ { 0x1E02, 0x42, 0x00, 0x00, 0x00 }, /* Ḃ to B */
+ { 0x1E03, 0x62, 0x00, 0x00, 0x00 }, /* ḃ to b */
+ { 0x1E0A, 0x44, 0x00, 0x00, 0x00 }, /* Ḋ to D */
+ { 0x1E0B, 0x64, 0x00, 0x00, 0x00 }, /* ḋ to d */
+ { 0x1E1E, 0x46, 0x00, 0x00, 0x00 }, /* Ḟ to F */
+ { 0x1E1F, 0x66, 0x00, 0x00, 0x00 }, /* ḟ to f */
+ { 0x1E40, 0x4D, 0x00, 0x00, 0x00 }, /* Ṁ to M */
+ { 0x1E41, 0x6D, 0x00, 0x00, 0x00 }, /* ṁ to m */
+ { 0x1E56, 0x50, 0x00, 0x00, 0x00 }, /* Ṗ to P */
+ { 0x1E57, 0x70, 0x00, 0x00, 0x00 }, /* ṗ to p */
+ { 0x1E60, 0x53, 0x00, 0x00, 0x00 }, /* Ṡ to S */
+ { 0x1E61, 0x73, 0x00, 0x00, 0x00 }, /* ṡ to s */
+ { 0x1E6A, 0x54, 0x00, 0x00, 0x00 }, /* Ṫ to T */
+ { 0x1E6B, 0x74, 0x00, 0x00, 0x00 }, /* ṫ to t */
+ { 0x1E80, 0x57, 0x00, 0x00, 0x00 }, /* Ẁ to W */
+ { 0x1E81, 0x77, 0x00, 0x00, 0x00 }, /* ẁ to w */
+ { 0x1E82, 0x57, 0x00, 0x00, 0x00 }, /* Ẃ to W */
+ { 0x1E83, 0x77, 0x00, 0x00, 0x00 }, /* ẃ to w */
+ { 0x1E84, 0x57, 0x00, 0x00, 0x00 }, /* Ẅ to W */
+ { 0x1E85, 0x77, 0x00, 0x00, 0x00 }, /* ẅ to w */
+ { 0x1EF2, 0x59, 0x00, 0x00, 0x00 }, /* Ỳ to Y */
+ { 0x1EF3, 0x79, 0x00, 0x00, 0x00 }, /* ỳ to y */
+ { 0xFB00, 0x66, 0x66, 0x00, 0x00 }, /* ff to ff */
+ { 0xFB01, 0x66, 0x69, 0x00, 0x00 }, /* fi to fi */
+ { 0xFB02, 0x66, 0x6C, 0x00, 0x00 }, /* fl to fl */
+ { 0xFB05, 0x73, 0x74, 0x00, 0x00 }, /* ſt to st */
+ { 0xFB06, 0x73, 0x74, 0x00, 0x00 }, /* st to st */
};
+
+static const Transliteration *spellfixFindTranslit(int c, int *pxTop){
+ *pxTop = (sizeof(translit)/sizeof(translit[0])) - 1;
+ return translit;
+}
/*
** Convert the input string from UTF-8 into pure ASCII by converting
** all non-ASCII characters to some combination of characters in the
** ASCII subset.
@@ -1605,11 +1709,15 @@
**
** Space to hold the returned string comes from sqlite3_malloc() and
** should be freed by the caller.
*/
static unsigned char *transliterate(const unsigned char *zIn, int nIn){
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+ unsigned char *zOut = sqlite3_malloc64( nIn*5 + 1 );
+#else
unsigned char *zOut = sqlite3_malloc64( nIn*4 + 1 );
+#endif
int c, sz, nOut;
if( zOut==0 ) return 0;
nOut = 0;
while( nIn>0 ){
c = utf8Read(zIn, nIn, &sz);
@@ -1617,27 +1725,33 @@
nIn -= sz;
if( c<=127 ){
zOut[nOut++] = (unsigned char)c;
}else{
int xTop, xBtm, x;
- xTop = sizeof(translit)/sizeof(translit[0]) - 1;
+ const Transliteration *tbl = spellfixFindTranslit(c, &xTop);
xBtm = 0;
while( xTop>=xBtm ){
x = (xTop + xBtm)/2;
- if( translit[x].cFrom==c ){
- zOut[nOut++] = translit[x].cTo0;
- if( translit[x].cTo1 ){
- zOut[nOut++] = translit[x].cTo1;
- /* Add an extra "ch" after the "sh" for Щ and щ */
- if( c==0x0429 || c== 0x0449 ){
- zOut[nOut++] = 'c';
- zOut[nOut++] = 'h';
+ if( tbl[x].cFrom==c ){
+ zOut[nOut++] = tbl[x].cTo0;
+ if( tbl[x].cTo1 ){
+ zOut[nOut++] = tbl[x].cTo1;
+ if( tbl[x].cTo2 ){
+ zOut[nOut++] = tbl[x].cTo2;
+ if( tbl[x].cTo3 ){
+ zOut[nOut++] = tbl[x].cTo3;
+#ifdef SQLITE_SPELLFIX_5BYTE_MAPPINGS
+ if( tbl[x].cTo4 ){
+ zOut[nOut++] = tbl[x].cTo4;
+ }
+#endif /* SQLITE_SPELLFIX_5BYTE_MAPPINGS */
+ }
}
}
c = 0;
break;
- }else if( translit[x].cFrom>c ){
+ }else if( tbl[x].cFrom>c ){
xTop = x-1;
}else{
xBtm = x+1;
}
}
@@ -1664,19 +1778,26 @@
i += sz;
nOut++;
if( c>=128 ){
int xTop, xBtm, x;
- xTop = sizeof(translit)/sizeof(translit[0]) - 1;
+ const Transliteration *tbl = spellfixFindTranslit(c, &xTop);
xBtm = 0;
while( xTop>=xBtm ){
x = (xTop + xBtm)/2;
- if( translit[x].cFrom==c ){
- if( translit[x].cTo1 ) nOut++;
- if( c==0x0429 || c== 0x0449 ) nOut += 2;
+ if( tbl[x].cFrom==c ){
+ if( tbl[x].cTo1 ){
+ nOut++;
+ if( tbl[x].cTo2 ){
+ nOut++;
+ if( tbl[x].cTo3 ){
+ nOut++;
+ }
+ }
+ }
break;
- }else if( translit[x].cFrom>c ){
+ }else if( tbl[x].cFrom>c ){
xTop = x-1;
}else{
xBtm = x+1;
}
}
@@ -2472,11 +2593,11 @@
goto filter_exit;
}
nPattern = (int)strlen(zPattern);
if( zPattern[nPattern-1]=='*' ) nPattern--;
zSql = sqlite3_mprintf(
- "SELECT id, word, rank, k1"
+ "SELECT id, word, rank, coalesce(k1,word)"
" FROM \"%w\".\"%w_vocab\""
" WHERE langid=%d AND k2>=?1 AND k2zDbName, p->zTableName, iLang
);
if( zSql==0 ){
@@ -2806,32 +2927,32 @@
}
if( sqlite3_value_type(argv[0])==SQLITE_NULL ){
if( sqlite3_value_type(argv[1])==SQLITE_NULL ){
spellfix1DbExec(&rc, db,
"INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) "
- "VALUES(%d,%d,%Q,%Q,%Q)",
+ "VALUES(%d,%d,%Q,nullif(%Q,%Q),%Q)",
p->zDbName, p->zTableName,
- iRank, iLang, zWord, zK1, zK2
+ iRank, iLang, zWord, zK1, zWord, zK2
);
}else{
newRowid = sqlite3_value_int64(argv[1]);
spellfix1DbExec(&rc, db,
"INSERT OR %s INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) "
- "VALUES(%lld,%d,%d,%Q,%Q,%Q)",
+ "VALUES(%lld,%d,%d,%Q,nullif(%Q,%Q),%Q)",
zConflict, p->zDbName, p->zTableName,
- newRowid, iRank, iLang, zWord, zK1, zK2
+ newRowid, iRank, iLang, zWord, zK1, zWord, zK2
);
}
*pRowid = sqlite3_last_insert_rowid(db);
}else{
rowid = sqlite3_value_int64(argv[0]);
newRowid = *pRowid = sqlite3_value_int64(argv[1]);
spellfix1DbExec(&rc, db,
"UPDATE OR %s \"%w\".\"%w_vocab\" SET id=%lld, rank=%d, langid=%d,"
- " word=%Q, k1=%Q, k2=%Q WHERE id=%lld",
+ " word=%Q, k1=nullif(%Q,%Q), k2=%Q WHERE id=%lld",
zConflict, p->zDbName, p->zTableName, newRowid, iRank, iLang,
- zWord, zK1, zK2, rowid
+ zWord, zK1, zWord, zK2, rowid
);
}
sqlite3_free(zK1);
sqlite3_free(zK2);
}
@@ -2893,22 +3014,26 @@
** Register the various functions and the virtual table.
*/
static int spellfix1Register(sqlite3 *db){
int rc = SQLITE_OK;
int i;
- rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0,
- transliterateSqlFunc, 0, 0);
+ rc = sqlite3_create_function(db, "spellfix1_translit", 1,
+ SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
+ transliterateSqlFunc, 0, 0);
if( rc==SQLITE_OK ){
- rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0,
+ rc = sqlite3_create_function(db, "spellfix1_editdist", 2,
+ SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
editdistSqlFunc, 0, 0);
}
if( rc==SQLITE_OK ){
- rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0,
+ rc = sqlite3_create_function(db, "spellfix1_phonehash", 1,
+ SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
phoneticHashSqlFunc, 0, 0);
}
if( rc==SQLITE_OK ){
- rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0,
+ rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1,
+ SQLITE_UTF8|SQLITE_DETERMINISTIC, 0,
scriptCodeSqlFunc, 0, 0);
}
if( rc==SQLITE_OK ){
rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0);
}
ADDED ext/misc/sqlar.c
Index: ext/misc/sqlar.c
==================================================================
--- /dev/null
+++ ext/misc/sqlar.c
@@ -0,0 +1,121 @@
+/*
+** 2017-12-17
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Utility functions sqlar_compress() and sqlar_uncompress(). Useful
+** for working with sqlar archives and used by the shell tool's built-in
+** sqlar support.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include
+
+/*
+** Implementation of the "sqlar_compress(X)" SQL function.
+**
+** If the type of X is SQLITE_BLOB, and compressing that blob using
+** zlib utility function compress() yields a smaller blob, return the
+** compressed blob. Otherwise, return a copy of X.
+**
+** SQLar uses the "zlib format" for compressed content. The zlib format
+** contains a two-byte identification header and a four-byte checksum at
+** the end. This is different from ZIP which uses the raw deflate format.
+**
+** Future enhancements to SQLar might add support for new compression formats.
+** If so, those new formats will be identified by alternative headers in the
+** compressed data.
+*/
+static void sqlarCompressFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ assert( argc==1 );
+ if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+ const Bytef *pData = sqlite3_value_blob(argv[0]);
+ uLong nData = sqlite3_value_bytes(argv[0]);
+ uLongf nOut = compressBound(nData);
+ Bytef *pOut;
+
+ pOut = (Bytef*)sqlite3_malloc(nOut);
+ if( pOut==0 ){
+ sqlite3_result_error_nomem(context);
+ return;
+ }else{
+ if( Z_OK!=compress(pOut, &nOut, pData, nData) ){
+ sqlite3_result_error(context, "error in compress()", -1);
+ }else if( nOut
+#include
+
+/* templatevtab_vtab is a subclass of sqlite3_vtab which is
+** underlying representation of the virtual table
+*/
+typedef struct templatevtab_vtab templatevtab_vtab;
+struct templatevtab_vtab {
+ sqlite3_vtab base; /* Base class - must be first */
+ /* Add new fields here, as necessary */
+};
+
+/* templatevtab_cursor is a subclass of sqlite3_vtab_cursor which will
+** serve as the underlying representation of a cursor that scans
+** over rows of the result
+*/
+typedef struct templatevtab_cursor templatevtab_cursor;
+struct templatevtab_cursor {
+ sqlite3_vtab_cursor base; /* Base class - must be first */
+ /* Insert new fields here. For this templatevtab we only keep track
+ ** of the rowid */
+ sqlite3_int64 iRowid; /* The rowid */
+};
+
+/*
+** The templatevtabConnect() method is invoked to create a new
+** template virtual table.
+**
+** Think of this routine as the constructor for templatevtab_vtab objects.
+**
+** All this routine needs to do is:
+**
+** (1) Allocate the templatevtab_vtab object and initialize all fields.
+**
+** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the
+** result set of queries against the virtual table will look like.
+*/
+static int templatevtabConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ templatevtab_vtab *pNew;
+ int rc;
+
+ rc = sqlite3_declare_vtab(db,
+ "CREATE TABLE x(a,b)"
+ );
+ /* For convenience, define symbolic names for the index to each column. */
+#define TEMPLATEVTAB_A 0
+#define TEMPLATEVTAB_B 1
+ if( rc==SQLITE_OK ){
+ pNew = sqlite3_malloc( sizeof(*pNew) );
+ *ppVtab = (sqlite3_vtab*)pNew;
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, sizeof(*pNew));
+ }
+ return rc;
+}
+
+/*
+** This method is the destructor for templatevtab_vtab objects.
+*/
+static int templatevtabDisconnect(sqlite3_vtab *pVtab){
+ templatevtab_vtab *p = (templatevtab_vtab*)pVtab;
+ sqlite3_free(p);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new templatevtab_cursor object.
+*/
+static int templatevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){
+ templatevtab_cursor *pCur;
+ pCur = sqlite3_malloc( sizeof(*pCur) );
+ if( pCur==0 ) return SQLITE_NOMEM;
+ memset(pCur, 0, sizeof(*pCur));
+ *ppCursor = &pCur->base;
+ return SQLITE_OK;
+}
+
+/*
+** Destructor for a templatevtab_cursor.
+*/
+static int templatevtabClose(sqlite3_vtab_cursor *cur){
+ templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+ sqlite3_free(pCur);
+ return SQLITE_OK;
+}
+
+
+/*
+** Advance a templatevtab_cursor to its next row of output.
+*/
+static int templatevtabNext(sqlite3_vtab_cursor *cur){
+ templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+ pCur->iRowid++;
+ return SQLITE_OK;
+}
+
+/*
+** Return values of columns for the row at which the templatevtab_cursor
+** is currently pointing.
+*/
+static int templatevtabColumn(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int i /* Which column to return */
+){
+ templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+ switch( i ){
+ case TEMPLATEVTAB_A:
+ sqlite3_result_int(ctx, 1000 + pCur->iRowid);
+ break;
+ default:
+ assert( i==TEMPLATEVTAB_B );
+ sqlite3_result_int(ctx, 2000 + pCur->iRowid);
+ break;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** Return the rowid for the current row. In this implementation, the
+** rowid is the same as the output value.
+*/
+static int templatevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+ templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+ *pRowid = pCur->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Return TRUE if the cursor has been moved off of the last
+** row of output.
+*/
+static int templatevtabEof(sqlite3_vtab_cursor *cur){
+ templatevtab_cursor *pCur = (templatevtab_cursor*)cur;
+ return pCur->iRowid>=10;
+}
+
+/*
+** This method is called to "rewind" the templatevtab_cursor object back
+** to the first row of output. This method is always called at least
+** once prior to any call to templatevtabColumn() or templatevtabRowid() or
+** templatevtabEof().
+*/
+static int templatevtabFilter(
+ sqlite3_vtab_cursor *pVtabCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ templatevtab_cursor *pCur = (templatevtab_cursor *)pVtabCursor;
+ pCur->iRowid = 1;
+ return SQLITE_OK;
+}
+
+/*
+** SQLite will invoke this method one or more times while planning a query
+** that uses the virtual table. This routine needs to create
+** a query plan for each invocation and compute an estimated cost for that
+** plan.
+*/
+static int templatevtabBestIndex(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ pIdxInfo->estimatedCost = (double)10;
+ pIdxInfo->estimatedRows = 10;
+ return SQLITE_OK;
+}
+
+/*
+** This following structure defines all the methods for the
+** virtual table.
+*/
+static sqlite3_module templatevtabModule = {
+ /* iVersion */ 0,
+ /* xCreate */ 0,
+ /* xConnect */ templatevtabConnect,
+ /* xBestIndex */ templatevtabBestIndex,
+ /* xDisconnect */ templatevtabDisconnect,
+ /* xDestroy */ 0,
+ /* xOpen */ templatevtabOpen,
+ /* xClose */ templatevtabClose,
+ /* xFilter */ templatevtabFilter,
+ /* xNext */ templatevtabNext,
+ /* xEof */ templatevtabEof,
+ /* xColumn */ templatevtabColumn,
+ /* xRowid */ templatevtabRowid,
+ /* xUpdate */ 0,
+ /* xBegin */ 0,
+ /* xSync */ 0,
+ /* xCommit */ 0,
+ /* xRollback */ 0,
+ /* xFindMethod */ 0,
+ /* xRename */ 0,
+ /* xSavepoint */ 0,
+ /* xRelease */ 0,
+ /* xRollbackTo */ 0,
+ /* xShadowName */ 0
+};
+
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_templatevtab_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ int rc = SQLITE_OK;
+ SQLITE_EXTENSION_INIT2(pApi);
+ rc = sqlite3_create_module(db, "templatevtab", &templatevtabModule, 0);
+ return rc;
+}
Index: ext/misc/unionvtab.c
==================================================================
--- ext/misc/unionvtab.c
+++ ext/misc/unionvtab.c
@@ -53,10 +53,12 @@
**
** 4. The largest rowid in the range of rowids that may be stored in the
** database table (an integer).
**
** SWARMVTAB
+**
+** LEGACY SYNTAX:
**
** A "swarmvtab" virtual table is created similarly to a unionvtab table:
**
** CREATE VIRTUAL TABLE
** USING swarmvtab(, );
@@ -64,17 +66,82 @@
** The difference is that for a swarmvtab table, the first column returned
** by the must return a path or URI that can be used to open
** the database file containing the source table. The option
** is optional. If included, it is the name of an application-defined
** SQL function that is invoked with the URI of the file, if the file
-** does not already exist on disk.
+** does not already exist on disk when required by swarmvtab.
+**
+** NEW SYNTAX:
+**
+** Using the new syntax, a swarmvtab table is created with:
+**
+** CREATE VIRTUAL TABLE USING swarmvtab(
+** [, ]
+** );
+**
+** where valid are:
+**
+** missing=
+** openclose=
+** maxopen=
+** =
+**
+** The must return the same 4 columns as for a swarmvtab
+** table in legacy mode. However, it may also return a 5th column - the
+** "context" column. The text value returned in this column is not used
+** at all by the swarmvtab implementation, except that it is passed as
+** an additional argument to the two UDF functions that may be invoked
+** (see below).
+**
+** The "missing" option, if present, specifies the name of an SQL UDF
+** function to be invoked if a database file is not already present on
+** disk when required by swarmvtab. If the did not provide
+** a context column, it is invoked as:
+**
+** SELECT ();
+**
+** Or, if there was a context column:
+**
+** SELECT (, );
+**
+** The "openclose" option may also specify a UDF function. This function
+** is invoked right before swarmvtab opens a database, and right after
+** it closes one. The first argument - or first two arguments, if
+** supplied the context column - is the same as for
+** the "missing" UDF. Following this, the UDF is passed integer value
+** 0 before a db is opened, and 1 right after it is closed. If both
+** a missing and openclose UDF is supplied, the application should expect
+** the following sequence of calls (for a single database):
+**
+** SELECT (, , 0);
+** if( db not already on disk ){
+** SELECT (, );
+** }
+** ... swarmvtab uses database ...
+** SELECT (, , 1);
+**
+** The "maxopen" option is used to configure the maximum number of
+** database files swarmvtab will hold open simultaneously (default 9).
+**
+** If an option name begins with a ":" character, then it is assumed
+** to be an SQL parameter. In this case, the specified text value is
+** bound to the same variable of the before it is
+** executed. It is an error of the named SQL parameter does not exist.
+** For example:
+**
+** CREATE VIRTUAL TABLE swarm USING swarmvtab(
+** 'SELECT :path || localfile, tbl, min, max FROM swarmdir',
+** :path='/home/user/databases/'
+** missing='missing_func'
+** );
*/
#include "sqlite3ext.h"
SQLITE_EXTENSION_INIT1
#include
#include
+#include
#ifndef SQLITE_OMIT_VIRTUALTABLE
/*
** Largest and smallest possible 64-bit signed integers. These macros
@@ -126,10 +193,11 @@
sqlite3_int64 iMin; /* Minimum rowid */
sqlite3_int64 iMax; /* Maximum rowid */
/* Fields used by swarmvtab only */
char *zFile; /* Database file containing table zTab */
+ char *zContext; /* Context string, if any */
int nUser; /* Current number of users */
sqlite3 *db; /* Database handle */
UnionSrc *pNextClosable; /* Next in list of closable sources */
};
@@ -143,12 +211,15 @@
int iPK; /* INTEGER PRIMARY KEY column, or -1 */
int nSrc; /* Number of elements in the aSrc[] array */
UnionSrc *aSrc; /* Array of source tables, sorted by rowid */
/* Used by swarmvtab only */
+ int bHasContext; /* Has context strings */
char *zSourceStr; /* Expected unionSourceToStr() value */
- char *zNotFoundCallback; /* UDF to invoke if file not found on open */
+ sqlite3_stmt *pNotFound; /* UDF to invoke if file not found on open */
+ sqlite3_stmt *pOpenClose; /* UDF to invoke on open and close */
+
UnionSrc *pClosable; /* First in list of closable sources */
int nOpen; /* Current number of open sources */
int nMaxOpen; /* Maximum number of open sources */
};
@@ -348,25 +419,61 @@
if( rc ){
*pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
}
}
}
+
+/*
+** If an "openclose" UDF was supplied when this virtual table was created,
+** invoke it now. The first argument passed is the name of the database
+** file for source pSrc. The second is integer value bClose.
+**
+** If successful, return SQLITE_OK. Otherwise an SQLite error code. In this
+** case if argument pzErr is not NULL, also set (*pzErr) to an English
+** language error message. The caller is responsible for eventually freeing
+** any error message using sqlite3_free().
+*/
+static int unionInvokeOpenClose(
+ UnionTab *pTab,
+ UnionSrc *pSrc,
+ int bClose,
+ char **pzErr
+){
+ int rc = SQLITE_OK;
+ if( pTab->pOpenClose ){
+ sqlite3_bind_text(pTab->pOpenClose, 1, pSrc->zFile, -1, SQLITE_STATIC);
+ if( pTab->bHasContext ){
+ sqlite3_bind_text(pTab->pOpenClose, 2, pSrc->zContext, -1, SQLITE_STATIC);
+ }
+ sqlite3_bind_int(pTab->pOpenClose, 2+pTab->bHasContext, bClose);
+ sqlite3_step(pTab->pOpenClose);
+ if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pOpenClose)) ){
+ if( pzErr ){
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+ }
+ }
+ }
+ return rc;
+}
/*
** This function is a no-op for unionvtab. For swarmvtab, it attempts to
** close open database files until at most nMax are open. An SQLite error
** code is returned if an error occurs, or SQLITE_OK otherwise.
*/
static void unionCloseSources(UnionTab *pTab, int nMax){
while( pTab->pClosable && pTab->nOpen>nMax ){
+ UnionSrc *p;
UnionSrc **pp;
for(pp=&pTab->pClosable; (*pp)->pNextClosable; pp=&(*pp)->pNextClosable);
- assert( (*pp)->db );
- sqlite3_close((*pp)->db);
- (*pp)->db = 0;
+ p = *pp;
+ assert( p->db );
+ sqlite3_close(p->db);
+ p->db = 0;
*pp = 0;
pTab->nOpen--;
+ unionInvokeOpenClose(pTab, p, 1, 0);
}
}
/*
** xDisconnect method.
@@ -375,17 +482,23 @@
if( pVtab ){
UnionTab *pTab = (UnionTab*)pVtab;
int i;
for(i=0; inSrc; i++){
UnionSrc *pSrc = &pTab->aSrc[i];
+ int bHaveSrcDb = (pSrc->db!=0);
+ sqlite3_close(pSrc->db);
+ if( bHaveSrcDb ){
+ unionInvokeOpenClose(pTab, pSrc, 1, 0);
+ }
sqlite3_free(pSrc->zDb);
sqlite3_free(pSrc->zTab);
sqlite3_free(pSrc->zFile);
- sqlite3_close(pSrc->db);
+ sqlite3_free(pSrc->zContext);
}
+ sqlite3_finalize(pTab->pNotFound);
+ sqlite3_finalize(pTab->pOpenClose);
sqlite3_free(pTab->zSourceStr);
- sqlite3_free(pTab->zNotFoundCallback);
sqlite3_free(pTab->aSrc);
sqlite3_free(pTab);
}
return SQLITE_OK;
}
@@ -494,33 +607,35 @@
sqlite3_free(z0);
return rc;
}
-
/*
** Try to open the swarmvtab database. If initially unable, invoke the
** not-found callback UDF and then try again.
*/
static int unionOpenDatabaseInner(UnionTab *pTab, UnionSrc *pSrc, char **pzErr){
- int rc = SQLITE_OK;
- static const int openFlags =
- SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+ static const int openFlags = SQLITE_OPEN_READONLY | SQLITE_OPEN_URI;
+ int rc;
+
+ rc = unionInvokeOpenClose(pTab, pSrc, 0, pzErr);
+ if( rc!=SQLITE_OK ) return rc;
+
rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
if( rc==SQLITE_OK ) return rc;
- if( pTab->zNotFoundCallback ){
- char *zSql = sqlite3_mprintf("SELECT \"%w\"(%Q);",
- pTab->zNotFoundCallback, pSrc->zFile);
+ if( pTab->pNotFound ){
sqlite3_close(pSrc->db);
pSrc->db = 0;
- if( zSql==0 ){
- *pzErr = sqlite3_mprintf("out of memory");
- return SQLITE_NOMEM;
+ sqlite3_bind_text(pTab->pNotFound, 1, pSrc->zFile, -1, SQLITE_STATIC);
+ if( pTab->bHasContext ){
+ sqlite3_bind_text(pTab->pNotFound, 2, pSrc->zContext, -1, SQLITE_STATIC);
}
- rc = sqlite3_exec(pTab->db, zSql, 0, 0, pzErr);
- sqlite3_free(zSql);
- if( rc ) return rc;
+ sqlite3_step(pTab->pNotFound);
+ if( SQLITE_OK!=(rc = sqlite3_reset(pTab->pNotFound)) ){
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+ return rc;
+ }
rc = sqlite3_open_v2(pSrc->zFile, &pSrc->db, openFlags, 0);
}
if( rc!=SQLITE_OK ){
*pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(pSrc->db));
}
@@ -570,10 +685,11 @@
pTab->pClosable = pSrc;
pTab->nOpen++;
}else{
sqlite3_close(pSrc->db);
pSrc->db = 0;
+ unionInvokeOpenClose(pTab, pSrc, 1, 0);
}
}
return rc;
}
@@ -624,10 +740,136 @@
unionCloseSources(pTab, pTab->nMaxOpen);
}
}
return rc;
}
+
+/*
+** Return true if the argument is a space, tab, CR or LF character.
+*/
+static int union_isspace(char c){
+ return (c==' ' || c=='\n' || c=='\r' || c=='\t');
+}
+
+/*
+** Return true if the argument is an alphanumeric character in the
+** ASCII range.
+*/
+static int union_isidchar(char c){
+ return ((c>='a' && c<='z') || (c>='A' && c<'Z') || (c>='0' && c<='9'));
+}
+
+/*
+** This function is called to handle all arguments following the first
+** (the SQL statement) passed to a swarmvtab (not unionvtab) CREATE
+** VIRTUAL TABLE statement. It may bind parameters to the SQL statement
+** or configure members of the UnionTab object passed as the second
+** argument.
+**
+** Refer to header comments at the top of this file for a description
+** of the arguments parsed.
+**
+** This function is a no-op if *pRc is other than SQLITE_OK when it is
+** called. Otherwise, if an error occurs, *pRc is set to an SQLite error
+** code. In this case *pzErr may be set to point to a buffer containing
+** an English language error message. It is the responsibility of the
+** caller to eventually free the buffer using sqlite3_free().
+*/
+static void unionConfigureVtab(
+ int *pRc, /* IN/OUT: Error code */
+ UnionTab *pTab, /* Table to configure */
+ sqlite3_stmt *pStmt, /* SQL statement to find sources */
+ int nArg, /* Number of entries in azArg[] array */
+ const char * const *azArg, /* Array of arguments to consider */
+ char **pzErr /* OUT: Error message */
+){
+ int rc = *pRc;
+ int i;
+ if( rc==SQLITE_OK ){
+ pTab->bHasContext = (sqlite3_column_count(pStmt)>4);
+ }
+ for(i=0; rc==SQLITE_OK && inMaxOpen = atoi(zVal);
+ if( pTab->nMaxOpen<=0 ){
+ *pzErr = sqlite3_mprintf("swarmvtab: illegal maxopen value");
+ rc = SQLITE_ERROR;
+ }
+ }else if( nOpt==7 && 0==sqlite3_strnicmp(zOpt, "missing", 7) ){
+ if( pTab->pNotFound ){
+ *pzErr = sqlite3_mprintf(
+ "swarmvtab: duplicate \"missing\" option");
+ rc = SQLITE_ERROR;
+ }else{
+ pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+ "SELECT \"%w\"(?%s)", zVal, pTab->bHasContext ? ",?" : ""
+ );
+ }
+ }else if( nOpt==9 && 0==sqlite3_strnicmp(zOpt, "openclose", 9) ){
+ if( pTab->pOpenClose ){
+ *pzErr = sqlite3_mprintf(
+ "swarmvtab: duplicate \"openclose\" option");
+ rc = SQLITE_ERROR;
+ }else{
+ pTab->pOpenClose = unionPreparePrintf(&rc, pzErr, pTab->db,
+ "SELECT \"%w\"(?,?%s)", zVal, pTab->bHasContext ? ",?" : ""
+ );
+ }
+ }else{
+ *pzErr = sqlite3_mprintf("swarmvtab: unrecognized option: %s",zOpt);
+ rc = SQLITE_ERROR;
+ }
+ sqlite3_free(zVal);
+ }
+ }else{
+ if( i==0 && nArg==1 ){
+ pTab->pNotFound = unionPreparePrintf(&rc, pzErr, pTab->db,
+ "SELECT \"%w\"(?)", zArg
+ );
+ }else{
+ *pzErr = sqlite3_mprintf( "swarmvtab: parse error: %s", azArg[i]);
+ rc = SQLITE_ERROR;
+ }
+ }
+ sqlite3_free(zArg);
+ }
+ }
+ *pRc = rc;
+}
/*
** xConnect/xCreate method.
**
** The argv[] array contains the following:
@@ -652,11 +894,11 @@
if( sqlite3_stricmp("temp", argv[1]) ){
/* unionvtab tables may only be created in the temp schema */
*pzErr = sqlite3_mprintf("%s tables must be created in TEMP schema", zVtab);
rc = SQLITE_ERROR;
- }else if( argc!=4 && argc!=5 ){
+ }else if( argc<4 || (argc>4 && bSwarm==0) ){
*pzErr = sqlite3_mprintf("wrong number of arguments for %s", zVtab);
rc = SQLITE_ERROR;
}else{
int nAlloc = 0; /* Allocated size of pTab->aSrc[] */
sqlite3_stmt *pStmt = 0; /* Argument statement */
@@ -671,10 +913,21 @@
"SELECT * FROM (%z) ORDER BY 3", zArg
);
/* Allocate the UnionTab structure */
pTab = unionMalloc(&rc, sizeof(UnionTab));
+ if( pTab ){
+ assert( rc==SQLITE_OK );
+ pTab->db = db;
+ pTab->bSwarm = bSwarm;
+ pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
+ }
+
+ /* Parse other CVT arguments, if any */
+ if( bSwarm ){
+ unionConfigureVtab(&rc, pTab, pStmt, argc-4, &argv[4], pzErr);
+ }
/* Iterate through the rows returned by the SQL statement specified
** as an argument to the CREATE VIRTUAL TABLE statement. */
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zDb = (const char*)sqlite3_column_text(pStmt, 0);
@@ -713,21 +966,19 @@
if( bSwarm ){
pSrc->zFile = unionStrdup(&rc, zDb);
}else{
pSrc->zDb = unionStrdup(&rc, zDb);
}
+ if( pTab->bHasContext ){
+ const char *zContext = (const char*)sqlite3_column_text(pStmt, 4);
+ pSrc->zContext = unionStrdup(&rc, zContext);
+ }
}
}
unionFinalize(&rc, pStmt, pzErr);
pStmt = 0;
- /* Capture the not-found callback UDF name */
- if( rc==SQLITE_OK && argc>=5 ){
- pTab->zNotFoundCallback = unionStrdup(&rc, argv[4]);
- unionDequote(pTab->zNotFoundCallback);
- }
-
/* It is an error if the SELECT statement returned zero rows. If only
** because there is no way to determine the schema of the virtual
** table in this case. */
if( rc==SQLITE_OK && pTab->nSrc==0 ){
*pzErr = sqlite3_mprintf("no source tables configured");
@@ -736,13 +987,10 @@
/* For unionvtab, verify that all source tables exist and have
** compatible schemas. For swarmvtab, attach the first database and
** check that the first table is a rowid table only. */
if( rc==SQLITE_OK ){
- pTab->db = db;
- pTab->bSwarm = bSwarm;
- pTab->nMaxOpen = SWARMVTAB_MAX_OPEN;
if( bSwarm ){
rc = unionOpenDatabase(pTab, 0, pzErr);
}else{
rc = unionSourceCheck(pTab, pzErr);
}
@@ -1100,11 +1348,12 @@
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
- 0 /* xRollbackTo */
+ 0, /* xRollbackTo */
+ 0 /* xShadowName */
};
int rc;
rc = sqlite3_create_module(db, "unionvtab", &unionModule, 0);
if( rc==SQLITE_OK ){
Index: ext/misc/vtablog.c
==================================================================
--- ext/misc/vtablog.c
+++ ext/misc/vtablog.c
@@ -490,10 +490,11 @@
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
+ 0, /* xShadowName */
};
#ifdef _WIN32
__declspec(dllexport)
#endif
ADDED ext/misc/zipfile.c
Index: ext/misc/zipfile.c
==================================================================
--- /dev/null
+++ ext/misc/zipfile.c
@@ -0,0 +1,2178 @@
+/*
+** 2017-12-26
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements a virtual table for reading and writing ZIP archive
+** files.
+**
+** Usage example:
+**
+** SELECT name, sz, datetime(mtime,'unixepoch') FROM zipfile($filename);
+**
+** Current limitations:
+**
+** * No support for encryption
+** * No support for ZIP archives spanning multiple files
+** * No support for zip64 extensions
+** * Only the "inflate/deflate" (zlib) compression method is supported
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include
+#include
+#include
+
+#include
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+#ifndef SQLITE_AMALGAMATION
+
+typedef sqlite3_int64 i64;
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned long u32;
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+
+#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST)
+# define ALWAYS(X) (1)
+# define NEVER(X) (0)
+#elif !defined(NDEBUG)
+# define ALWAYS(X) ((X)?1:(assert(0),0))
+# define NEVER(X) ((X)?(assert(0),1):0)
+#else
+# define ALWAYS(X) (X)
+# define NEVER(X) (X)
+#endif
+
+#endif /* SQLITE_AMALGAMATION */
+
+/*
+** Definitions for mode bitmasks S_IFDIR, S_IFREG and S_IFLNK.
+**
+** In some ways it would be better to obtain these values from system
+** header files. But, the dependency is undesirable and (a) these
+** have been stable for decades, (b) the values are part of POSIX and
+** are also made explicit in [man stat], and (c) are part of the
+** file format for zip archives.
+*/
+#ifndef S_IFDIR
+# define S_IFDIR 0040000
+#endif
+#ifndef S_IFREG
+# define S_IFREG 0100000
+#endif
+#ifndef S_IFLNK
+# define S_IFLNK 0120000
+#endif
+
+static const char ZIPFILE_SCHEMA[] =
+ "CREATE TABLE y("
+ "name PRIMARY KEY," /* 0: Name of file in zip archive */
+ "mode," /* 1: POSIX mode for file */
+ "mtime," /* 2: Last modification time (secs since 1970)*/
+ "sz," /* 3: Size of object */
+ "rawdata," /* 4: Raw data */
+ "data," /* 5: Uncompressed data */
+ "method," /* 6: Compression method (integer) */
+ "z HIDDEN" /* 7: Name of zip file */
+ ") WITHOUT ROWID;";
+
+#define ZIPFILE_F_COLUMN_IDX 7 /* Index of column "file" in the above */
+#define ZIPFILE_BUFFER_SIZE (64*1024)
+
+
+/*
+** Magic numbers used to read and write zip files.
+**
+** ZIPFILE_NEWENTRY_MADEBY:
+** Use this value for the "version-made-by" field in new zip file
+** entries. The upper byte indicates "unix", and the lower byte
+** indicates that the zip file matches pkzip specification 3.0.
+** This is what info-zip seems to do.
+**
+** ZIPFILE_NEWENTRY_REQUIRED:
+** Value for "version-required-to-extract" field of new entries.
+** Version 2.0 is required to support folders and deflate compression.
+**
+** ZIPFILE_NEWENTRY_FLAGS:
+** Value for "general-purpose-bit-flags" field of new entries. Bit
+** 11 means "utf-8 filename and comment".
+**
+** ZIPFILE_SIGNATURE_CDS:
+** First 4 bytes of a valid CDS record.
+**
+** ZIPFILE_SIGNATURE_LFH:
+** First 4 bytes of a valid LFH record.
+**
+** ZIPFILE_SIGNATURE_EOCD
+** First 4 bytes of a valid EOCD record.
+*/
+#define ZIPFILE_EXTRA_TIMESTAMP 0x5455
+#define ZIPFILE_NEWENTRY_MADEBY ((3<<8) + 30)
+#define ZIPFILE_NEWENTRY_REQUIRED 20
+#define ZIPFILE_NEWENTRY_FLAGS 0x800
+#define ZIPFILE_SIGNATURE_CDS 0x02014b50
+#define ZIPFILE_SIGNATURE_LFH 0x04034b50
+#define ZIPFILE_SIGNATURE_EOCD 0x06054b50
+
+/*
+** The sizes of the fixed-size part of each of the three main data
+** structures in a zip archive.
+*/
+#define ZIPFILE_LFH_FIXED_SZ 30
+#define ZIPFILE_EOCD_FIXED_SZ 22
+#define ZIPFILE_CDS_FIXED_SZ 46
+
+/*
+*** 4.3.16 End of central directory record:
+***
+*** end of central dir signature 4 bytes (0x06054b50)
+*** number of this disk 2 bytes
+*** number of the disk with the
+*** start of the central directory 2 bytes
+*** total number of entries in the
+*** central directory on this disk 2 bytes
+*** total number of entries in
+*** the central directory 2 bytes
+*** size of the central directory 4 bytes
+*** offset of start of central
+*** directory with respect to
+*** the starting disk number 4 bytes
+*** .ZIP file comment length 2 bytes
+*** .ZIP file comment (variable size)
+*/
+typedef struct ZipfileEOCD ZipfileEOCD;
+struct ZipfileEOCD {
+ u16 iDisk;
+ u16 iFirstDisk;
+ u16 nEntry;
+ u16 nEntryTotal;
+ u32 nSize;
+ u32 iOffset;
+};
+
+/*
+*** 4.3.12 Central directory structure:
+***
+*** ...
+***
+*** central file header signature 4 bytes (0x02014b50)
+*** version made by 2 bytes
+*** version needed to extract 2 bytes
+*** general purpose bit flag 2 bytes
+*** compression method 2 bytes
+*** last mod file time 2 bytes
+*** last mod file date 2 bytes
+*** crc-32 4 bytes
+*** compressed size 4 bytes
+*** uncompressed size 4 bytes
+*** file name length 2 bytes
+*** extra field length 2 bytes
+*** file comment length 2 bytes
+*** disk number start 2 bytes
+*** internal file attributes 2 bytes
+*** external file attributes 4 bytes
+*** relative offset of local header 4 bytes
+*/
+typedef struct ZipfileCDS ZipfileCDS;
+struct ZipfileCDS {
+ u16 iVersionMadeBy;
+ u16 iVersionExtract;
+ u16 flags;
+ u16 iCompression;
+ u16 mTime;
+ u16 mDate;
+ u32 crc32;
+ u32 szCompressed;
+ u32 szUncompressed;
+ u16 nFile;
+ u16 nExtra;
+ u16 nComment;
+ u16 iDiskStart;
+ u16 iInternalAttr;
+ u32 iExternalAttr;
+ u32 iOffset;
+ char *zFile; /* Filename (sqlite3_malloc()) */
+};
+
+/*
+*** 4.3.7 Local file header:
+***
+*** local file header signature 4 bytes (0x04034b50)
+*** version needed to extract 2 bytes
+*** general purpose bit flag 2 bytes
+*** compression method 2 bytes
+*** last mod file time 2 bytes
+*** last mod file date 2 bytes
+*** crc-32 4 bytes
+*** compressed size 4 bytes
+*** uncompressed size 4 bytes
+*** file name length 2 bytes
+*** extra field length 2 bytes
+***
+*/
+typedef struct ZipfileLFH ZipfileLFH;
+struct ZipfileLFH {
+ u16 iVersionExtract;
+ u16 flags;
+ u16 iCompression;
+ u16 mTime;
+ u16 mDate;
+ u32 crc32;
+ u32 szCompressed;
+ u32 szUncompressed;
+ u16 nFile;
+ u16 nExtra;
+};
+
+typedef struct ZipfileEntry ZipfileEntry;
+struct ZipfileEntry {
+ ZipfileCDS cds; /* Parsed CDS record */
+ u32 mUnixTime; /* Modification time, in UNIX format */
+ u8 *aExtra; /* cds.nExtra+cds.nComment bytes of extra data */
+ i64 iDataOff; /* Offset to data in file (if aData==0) */
+ u8 *aData; /* cds.szCompressed bytes of compressed data */
+ ZipfileEntry *pNext; /* Next element in in-memory CDS */
+};
+
+/*
+** Cursor type for zipfile tables.
+*/
+typedef struct ZipfileCsr ZipfileCsr;
+struct ZipfileCsr {
+ sqlite3_vtab_cursor base; /* Base class - must be first */
+ i64 iId; /* Cursor ID */
+ u8 bEof; /* True when at EOF */
+ u8 bNoop; /* If next xNext() call is no-op */
+
+ /* Used outside of write transactions */
+ FILE *pFile; /* Zip file */
+ i64 iNextOff; /* Offset of next record in central directory */
+ ZipfileEOCD eocd; /* Parse of central directory record */
+
+ ZipfileEntry *pFreeEntry; /* Free this list when cursor is closed or reset */
+ ZipfileEntry *pCurrent; /* Current entry */
+ ZipfileCsr *pCsrNext; /* Next cursor on same virtual table */
+};
+
+typedef struct ZipfileTab ZipfileTab;
+struct ZipfileTab {
+ sqlite3_vtab base; /* Base class - must be first */
+ char *zFile; /* Zip file this table accesses (may be NULL) */
+ sqlite3 *db; /* Host database connection */
+ u8 *aBuffer; /* Temporary buffer used for various tasks */
+
+ ZipfileCsr *pCsrList; /* List of cursors */
+ i64 iNextCsrid;
+
+ /* The following are used by write transactions only */
+ ZipfileEntry *pFirstEntry; /* Linked list of all files (if pWriteFd!=0) */
+ ZipfileEntry *pLastEntry; /* Last element in pFirstEntry list */
+ FILE *pWriteFd; /* File handle open on zip archive */
+ i64 szCurrent; /* Current size of zip archive */
+ i64 szOrig; /* Size of archive at start of transaction */
+};
+
+/*
+** Set the error message contained in context ctx to the results of
+** vprintf(zFmt, ...).
+*/
+static void zipfileCtxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){
+ char *zMsg = 0;
+ va_list ap;
+ va_start(ap, zFmt);
+ zMsg = sqlite3_vmprintf(zFmt, ap);
+ sqlite3_result_error(ctx, zMsg, -1);
+ sqlite3_free(zMsg);
+ va_end(ap);
+}
+
+/*
+** If string zIn is quoted, dequote it in place. Otherwise, if the string
+** is not quoted, do nothing.
+*/
+static void zipfileDequote(char *zIn){
+ char q = zIn[0];
+ if( q=='"' || q=='\'' || q=='`' || q=='[' ){
+ int iIn = 1;
+ int iOut = 0;
+ if( q=='[' ) q = ']';
+ while( ALWAYS(zIn[iIn]) ){
+ char c = zIn[iIn++];
+ if( c==q && zIn[iIn++]!=q ) break;
+ zIn[iOut++] = c;
+ }
+ zIn[iOut] = '\0';
+ }
+}
+
+/*
+** Construct a new ZipfileTab virtual table object.
+**
+** argv[0] -> module name ("zipfile")
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> "column name" and other module argument fields.
+*/
+static int zipfileConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ int nByte = sizeof(ZipfileTab) + ZIPFILE_BUFFER_SIZE;
+ int nFile = 0;
+ const char *zFile = 0;
+ ZipfileTab *pNew = 0;
+ int rc;
+
+ /* If the table name is not "zipfile", require that the argument be
+ ** specified. This stops zipfile tables from being created as:
+ **
+ ** CREATE VIRTUAL TABLE zzz USING zipfile();
+ **
+ ** It does not prevent:
+ **
+ ** CREATE VIRTUAL TABLE zipfile USING zipfile();
+ */
+ assert( 0==sqlite3_stricmp(argv[0], "zipfile") );
+ if( (0!=sqlite3_stricmp(argv[2], "zipfile") && argc<4) || argc>4 ){
+ *pzErr = sqlite3_mprintf("zipfile constructor requires one argument");
+ return SQLITE_ERROR;
+ }
+
+ if( argc>3 ){
+ zFile = argv[3];
+ nFile = (int)strlen(zFile)+1;
+ }
+
+ rc = sqlite3_declare_vtab(db, ZIPFILE_SCHEMA);
+ if( rc==SQLITE_OK ){
+ pNew = (ZipfileTab*)sqlite3_malloc(nByte+nFile);
+ if( pNew==0 ) return SQLITE_NOMEM;
+ memset(pNew, 0, nByte+nFile);
+ pNew->db = db;
+ pNew->aBuffer = (u8*)&pNew[1];
+ if( zFile ){
+ pNew->zFile = (char*)&pNew->aBuffer[ZIPFILE_BUFFER_SIZE];
+ memcpy(pNew->zFile, zFile, nFile);
+ zipfileDequote(pNew->zFile);
+ }
+ }
+ *ppVtab = (sqlite3_vtab*)pNew;
+ return rc;
+}
+
+/*
+** Free the ZipfileEntry structure indicated by the only argument.
+*/
+static void zipfileEntryFree(ZipfileEntry *p){
+ if( p ){
+ sqlite3_free(p->cds.zFile);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Release resources that should be freed at the end of a write
+** transaction.
+*/
+static void zipfileCleanupTransaction(ZipfileTab *pTab){
+ ZipfileEntry *pEntry;
+ ZipfileEntry *pNext;
+
+ if( pTab->pWriteFd ){
+ fclose(pTab->pWriteFd);
+ pTab->pWriteFd = 0;
+ }
+ for(pEntry=pTab->pFirstEntry; pEntry; pEntry=pNext){
+ pNext = pEntry->pNext;
+ zipfileEntryFree(pEntry);
+ }
+ pTab->pFirstEntry = 0;
+ pTab->pLastEntry = 0;
+ pTab->szCurrent = 0;
+ pTab->szOrig = 0;
+}
+
+/*
+** This method is the destructor for zipfile vtab objects.
+*/
+static int zipfileDisconnect(sqlite3_vtab *pVtab){
+ zipfileCleanupTransaction((ZipfileTab*)pVtab);
+ sqlite3_free(pVtab);
+ return SQLITE_OK;
+}
+
+/*
+** Constructor for a new ZipfileCsr object.
+*/
+static int zipfileOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){
+ ZipfileTab *pTab = (ZipfileTab*)p;
+ ZipfileCsr *pCsr;
+ pCsr = sqlite3_malloc(sizeof(*pCsr));
+ *ppCsr = (sqlite3_vtab_cursor*)pCsr;
+ if( pCsr==0 ){
+ return SQLITE_NOMEM;
+ }
+ memset(pCsr, 0, sizeof(*pCsr));
+ pCsr->iId = ++pTab->iNextCsrid;
+ pCsr->pCsrNext = pTab->pCsrList;
+ pTab->pCsrList = pCsr;
+ return SQLITE_OK;
+}
+
+/*
+** Reset a cursor back to the state it was in when first returned
+** by zipfileOpen().
+*/
+static void zipfileResetCursor(ZipfileCsr *pCsr){
+ ZipfileEntry *p;
+ ZipfileEntry *pNext;
+
+ pCsr->bEof = 0;
+ if( pCsr->pFile ){
+ fclose(pCsr->pFile);
+ pCsr->pFile = 0;
+ zipfileEntryFree(pCsr->pCurrent);
+ pCsr->pCurrent = 0;
+ }
+
+ for(p=pCsr->pFreeEntry; p; p=pNext){
+ pNext = p->pNext;
+ zipfileEntryFree(p);
+ }
+}
+
+/*
+** Destructor for an ZipfileCsr.
+*/
+static int zipfileClose(sqlite3_vtab_cursor *cur){
+ ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+ ZipfileTab *pTab = (ZipfileTab*)(pCsr->base.pVtab);
+ ZipfileCsr **pp;
+ zipfileResetCursor(pCsr);
+
+ /* Remove this cursor from the ZipfileTab.pCsrList list. */
+ for(pp=&pTab->pCsrList; *pp!=pCsr; pp=&((*pp)->pCsrNext));
+ *pp = pCsr->pCsrNext;
+
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Set the error message for the virtual table associated with cursor
+** pCsr to the results of vprintf(zFmt, ...).
+*/
+static void zipfileTableErr(ZipfileTab *pTab, const char *zFmt, ...){
+ va_list ap;
+ va_start(ap, zFmt);
+ sqlite3_free(pTab->base.zErrMsg);
+ pTab->base.zErrMsg = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+}
+static void zipfileCursorErr(ZipfileCsr *pCsr, const char *zFmt, ...){
+ va_list ap;
+ va_start(ap, zFmt);
+ sqlite3_free(pCsr->base.pVtab->zErrMsg);
+ pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+}
+
+/*
+** Read nRead bytes of data from offset iOff of file pFile into buffer
+** aRead[]. Return SQLITE_OK if successful, or an SQLite error code
+** otherwise.
+**
+** If an error does occur, output variable (*pzErrmsg) may be set to point
+** to an English language error message. It is the responsibility of the
+** caller to eventually free this buffer using
+** sqlite3_free().
+*/
+static int zipfileReadData(
+ FILE *pFile, /* Read from this file */
+ u8 *aRead, /* Read into this buffer */
+ int nRead, /* Number of bytes to read */
+ i64 iOff, /* Offset to read from */
+ char **pzErrmsg /* OUT: Error message (from sqlite3_malloc) */
+){
+ size_t n;
+ fseek(pFile, (long)iOff, SEEK_SET);
+ n = fread(aRead, 1, nRead, pFile);
+ if( (int)n!=nRead ){
+ *pzErrmsg = sqlite3_mprintf("error in fread()");
+ return SQLITE_ERROR;
+ }
+ return SQLITE_OK;
+}
+
+static int zipfileAppendData(
+ ZipfileTab *pTab,
+ const u8 *aWrite,
+ int nWrite
+){
+ size_t n;
+ fseek(pTab->pWriteFd, (long)pTab->szCurrent, SEEK_SET);
+ n = fwrite(aWrite, 1, nWrite, pTab->pWriteFd);
+ if( (int)n!=nWrite ){
+ pTab->base.zErrMsg = sqlite3_mprintf("error in fwrite()");
+ return SQLITE_ERROR;
+ }
+ pTab->szCurrent += nWrite;
+ return SQLITE_OK;
+}
+
+/*
+** Read and return a 16-bit little-endian unsigned integer from buffer aBuf.
+*/
+static u16 zipfileGetU16(const u8 *aBuf){
+ return (aBuf[1] << 8) + aBuf[0];
+}
+
+/*
+** Read and return a 32-bit little-endian unsigned integer from buffer aBuf.
+*/
+static u32 zipfileGetU32(const u8 *aBuf){
+ return ((u32)(aBuf[3]) << 24)
+ + ((u32)(aBuf[2]) << 16)
+ + ((u32)(aBuf[1]) << 8)
+ + ((u32)(aBuf[0]) << 0);
+}
+
+/*
+** Write a 16-bit little endiate integer into buffer aBuf.
+*/
+static void zipfilePutU16(u8 *aBuf, u16 val){
+ aBuf[0] = val & 0xFF;
+ aBuf[1] = (val>>8) & 0xFF;
+}
+
+/*
+** Write a 32-bit little endiate integer into buffer aBuf.
+*/
+static void zipfilePutU32(u8 *aBuf, u32 val){
+ aBuf[0] = val & 0xFF;
+ aBuf[1] = (val>>8) & 0xFF;
+ aBuf[2] = (val>>16) & 0xFF;
+ aBuf[3] = (val>>24) & 0xFF;
+}
+
+#define zipfileRead32(aBuf) ( aBuf+=4, zipfileGetU32(aBuf-4) )
+#define zipfileRead16(aBuf) ( aBuf+=2, zipfileGetU16(aBuf-2) )
+
+#define zipfileWrite32(aBuf,val) { zipfilePutU32(aBuf,val); aBuf+=4; }
+#define zipfileWrite16(aBuf,val) { zipfilePutU16(aBuf,val); aBuf+=2; }
+
+/*
+** Magic numbers used to read CDS records.
+*/
+#define ZIPFILE_CDS_NFILE_OFF 28
+#define ZIPFILE_CDS_SZCOMPRESSED_OFF 20
+
+/*
+** Decode the CDS record in buffer aBuf into (*pCDS). Return SQLITE_ERROR
+** if the record is not well-formed, or SQLITE_OK otherwise.
+*/
+static int zipfileReadCDS(u8 *aBuf, ZipfileCDS *pCDS){
+ u8 *aRead = aBuf;
+ u32 sig = zipfileRead32(aRead);
+ int rc = SQLITE_OK;
+ if( sig!=ZIPFILE_SIGNATURE_CDS ){
+ rc = SQLITE_ERROR;
+ }else{
+ pCDS->iVersionMadeBy = zipfileRead16(aRead);
+ pCDS->iVersionExtract = zipfileRead16(aRead);
+ pCDS->flags = zipfileRead16(aRead);
+ pCDS->iCompression = zipfileRead16(aRead);
+ pCDS->mTime = zipfileRead16(aRead);
+ pCDS->mDate = zipfileRead16(aRead);
+ pCDS->crc32 = zipfileRead32(aRead);
+ pCDS->szCompressed = zipfileRead32(aRead);
+ pCDS->szUncompressed = zipfileRead32(aRead);
+ assert( aRead==&aBuf[ZIPFILE_CDS_NFILE_OFF] );
+ pCDS->nFile = zipfileRead16(aRead);
+ pCDS->nExtra = zipfileRead16(aRead);
+ pCDS->nComment = zipfileRead16(aRead);
+ pCDS->iDiskStart = zipfileRead16(aRead);
+ pCDS->iInternalAttr = zipfileRead16(aRead);
+ pCDS->iExternalAttr = zipfileRead32(aRead);
+ pCDS->iOffset = zipfileRead32(aRead);
+ assert( aRead==&aBuf[ZIPFILE_CDS_FIXED_SZ] );
+ }
+
+ return rc;
+}
+
+/*
+** Decode the LFH record in buffer aBuf into (*pLFH). Return SQLITE_ERROR
+** if the record is not well-formed, or SQLITE_OK otherwise.
+*/
+static int zipfileReadLFH(
+ u8 *aBuffer,
+ ZipfileLFH *pLFH
+){
+ u8 *aRead = aBuffer;
+ int rc = SQLITE_OK;
+
+ u32 sig = zipfileRead32(aRead);
+ if( sig!=ZIPFILE_SIGNATURE_LFH ){
+ rc = SQLITE_ERROR;
+ }else{
+ pLFH->iVersionExtract = zipfileRead16(aRead);
+ pLFH->flags = zipfileRead16(aRead);
+ pLFH->iCompression = zipfileRead16(aRead);
+ pLFH->mTime = zipfileRead16(aRead);
+ pLFH->mDate = zipfileRead16(aRead);
+ pLFH->crc32 = zipfileRead32(aRead);
+ pLFH->szCompressed = zipfileRead32(aRead);
+ pLFH->szUncompressed = zipfileRead32(aRead);
+ pLFH->nFile = zipfileRead16(aRead);
+ pLFH->nExtra = zipfileRead16(aRead);
+ }
+ return rc;
+}
+
+
+/*
+** Buffer aExtra (size nExtra bytes) contains zip archive "extra" fields.
+** Scan through this buffer to find an "extra-timestamp" field. If one
+** exists, extract the 32-bit modification-timestamp from it and store
+** the value in output parameter *pmTime.
+**
+** Zero is returned if no extra-timestamp record could be found (and so
+** *pmTime is left unchanged), or non-zero otherwise.
+**
+** The general format of an extra field is:
+**
+** Header ID 2 bytes
+** Data Size 2 bytes
+** Data N bytes
+*/
+static int zipfileScanExtra(u8 *aExtra, int nExtra, u32 *pmTime){
+ int ret = 0;
+ u8 *p = aExtra;
+ u8 *pEnd = &aExtra[nExtra];
+
+ while( p modtime is present */
+ *pmTime = zipfileGetU32(&p[1]);
+ ret = 1;
+ }
+ break;
+ }
+ }
+
+ p += nByte;
+ }
+ return ret;
+}
+
+/*
+** Convert the standard MS-DOS timestamp stored in the mTime and mDate
+** fields of the CDS structure passed as the only argument to a 32-bit
+** UNIX seconds-since-the-epoch timestamp. Return the result.
+**
+** "Standard" MS-DOS time format:
+**
+** File modification time:
+** Bits 00-04: seconds divided by 2
+** Bits 05-10: minute
+** Bits 11-15: hour
+** File modification date:
+** Bits 00-04: day
+** Bits 05-08: month (1-12)
+** Bits 09-15: years from 1980
+**
+** https://msdn.microsoft.com/en-us/library/9kkf9tah.aspx
+*/
+static u32 zipfileMtime(ZipfileCDS *pCDS){
+ int Y = (1980 + ((pCDS->mDate >> 9) & 0x7F));
+ int M = ((pCDS->mDate >> 5) & 0x0F);
+ int D = (pCDS->mDate & 0x1F);
+ int B = -13;
+
+ int sec = (pCDS->mTime & 0x1F)*2;
+ int min = (pCDS->mTime >> 5) & 0x3F;
+ int hr = (pCDS->mTime >> 11) & 0x1F;
+ i64 JD;
+
+ /* JD = INT(365.25 * (Y+4716)) + INT(30.6001 * (M+1)) + D + B - 1524.5 */
+
+ /* Calculate the JD in seconds for noon on the day in question */
+ if( M<3 ){
+ Y = Y-1;
+ M = M+12;
+ }
+ JD = (i64)(24*60*60) * (
+ (int)(365.25 * (Y + 4716))
+ + (int)(30.6001 * (M + 1))
+ + D + B - 1524
+ );
+
+ /* Correct the JD for the time within the day */
+ JD += (hr-12) * 3600 + min * 60 + sec;
+
+ /* Convert JD to unix timestamp (the JD epoch is 2440587.5) */
+ return (u32)(JD - (i64)(24405875) * 24*60*6);
+}
+
+/*
+** The opposite of zipfileMtime(). This function populates the mTime and
+** mDate fields of the CDS structure passed as the first argument according
+** to the UNIX timestamp value passed as the second.
+*/
+static void zipfileMtimeToDos(ZipfileCDS *pCds, u32 mUnixTime){
+ /* Convert unix timestamp to JD (2440588 is noon on 1/1/1970) */
+ i64 JD = (i64)2440588 + mUnixTime / (24*60*60);
+
+ int A, B, C, D, E;
+ int yr, mon, day;
+ int hr, min, sec;
+
+ A = (int)((JD - 1867216.25)/36524.25);
+ A = (int)(JD + 1 + A - (A/4));
+ B = A + 1524;
+ C = (int)((B - 122.1)/365.25);
+ D = (36525*(C&32767))/100;
+ E = (int)((B-D)/30.6001);
+
+ day = B - D - (int)(30.6001*E);
+ mon = (E<14 ? E-1 : E-13);
+ yr = mon>2 ? C-4716 : C-4715;
+
+ hr = (mUnixTime % (24*60*60)) / (60*60);
+ min = (mUnixTime % (60*60)) / 60;
+ sec = (mUnixTime % 60);
+
+ if( yr>=1980 ){
+ pCds->mDate = (u16)(day + (mon << 5) + ((yr-1980) << 9));
+ pCds->mTime = (u16)(sec/2 + (min<<5) + (hr<<11));
+ }else{
+ pCds->mDate = pCds->mTime = 0;
+ }
+
+ assert( mUnixTime<315507600
+ || mUnixTime==zipfileMtime(pCds)
+ || ((mUnixTime % 2) && mUnixTime-1==zipfileMtime(pCds))
+ /* || (mUnixTime % 2) */
+ );
+}
+
+/*
+** If aBlob is not NULL, then it is a pointer to a buffer (nBlob bytes in
+** size) containing an entire zip archive image. Or, if aBlob is NULL,
+** then pFile is a file-handle open on a zip file. In either case, this
+** function creates a ZipfileEntry object based on the zip archive entry
+** for which the CDS record is at offset iOff.
+**
+** If successful, SQLITE_OK is returned and (*ppEntry) set to point to
+** the new object. Otherwise, an SQLite error code is returned and the
+** final value of (*ppEntry) undefined.
+*/
+static int zipfileGetEntry(
+ ZipfileTab *pTab, /* Store any error message here */
+ const u8 *aBlob, /* Pointer to in-memory file image */
+ int nBlob, /* Size of aBlob[] in bytes */
+ FILE *pFile, /* If aBlob==0, read from this file */
+ i64 iOff, /* Offset of CDS record */
+ ZipfileEntry **ppEntry /* OUT: Pointer to new object */
+){
+ u8 *aRead;
+ char **pzErr = &pTab->base.zErrMsg;
+ int rc = SQLITE_OK;
+
+ if( aBlob==0 ){
+ aRead = pTab->aBuffer;
+ rc = zipfileReadData(pFile, aRead, ZIPFILE_CDS_FIXED_SZ, iOff, pzErr);
+ }else{
+ aRead = (u8*)&aBlob[iOff];
+ }
+
+ if( rc==SQLITE_OK ){
+ int nAlloc;
+ ZipfileEntry *pNew;
+
+ int nFile = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF]);
+ int nExtra = zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+2]);
+ nExtra += zipfileGetU16(&aRead[ZIPFILE_CDS_NFILE_OFF+4]);
+
+ nAlloc = sizeof(ZipfileEntry) + nExtra;
+ if( aBlob ){
+ nAlloc += zipfileGetU32(&aRead[ZIPFILE_CDS_SZCOMPRESSED_OFF]);
+ }
+
+ pNew = (ZipfileEntry*)sqlite3_malloc(nAlloc);
+ if( pNew==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pNew, 0, sizeof(ZipfileEntry));
+ rc = zipfileReadCDS(aRead, &pNew->cds);
+ if( rc!=SQLITE_OK ){
+ *pzErr = sqlite3_mprintf("failed to read CDS at offset %lld", iOff);
+ }else if( aBlob==0 ){
+ rc = zipfileReadData(
+ pFile, aRead, nExtra+nFile, iOff+ZIPFILE_CDS_FIXED_SZ, pzErr
+ );
+ }else{
+ aRead = (u8*)&aBlob[iOff + ZIPFILE_CDS_FIXED_SZ];
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ u32 *pt = &pNew->mUnixTime;
+ pNew->cds.zFile = sqlite3_mprintf("%.*s", nFile, aRead);
+ pNew->aExtra = (u8*)&pNew[1];
+ memcpy(pNew->aExtra, &aRead[nFile], nExtra);
+ if( pNew->cds.zFile==0 ){
+ rc = SQLITE_NOMEM;
+ }else if( 0==zipfileScanExtra(&aRead[nFile], pNew->cds.nExtra, pt) ){
+ pNew->mUnixTime = zipfileMtime(&pNew->cds);
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ static const int szFix = ZIPFILE_LFH_FIXED_SZ;
+ ZipfileLFH lfh;
+ if( pFile ){
+ rc = zipfileReadData(pFile, aRead, szFix, pNew->cds.iOffset, pzErr);
+ }else{
+ aRead = (u8*)&aBlob[pNew->cds.iOffset];
+ }
+
+ rc = zipfileReadLFH(aRead, &lfh);
+ if( rc==SQLITE_OK ){
+ pNew->iDataOff = pNew->cds.iOffset + ZIPFILE_LFH_FIXED_SZ;
+ pNew->iDataOff += lfh.nFile + lfh.nExtra;
+ if( aBlob && pNew->cds.szCompressed ){
+ pNew->aData = &pNew->aExtra[nExtra];
+ memcpy(pNew->aData, &aBlob[pNew->iDataOff], pNew->cds.szCompressed);
+ }
+ }else{
+ *pzErr = sqlite3_mprintf("failed to read LFH at offset %d",
+ (int)pNew->cds.iOffset
+ );
+ }
+ }
+
+ if( rc!=SQLITE_OK ){
+ zipfileEntryFree(pNew);
+ }else{
+ *ppEntry = pNew;
+ }
+ }
+
+ return rc;
+}
+
+/*
+** Advance an ZipfileCsr to its next row of output.
+*/
+static int zipfileNext(sqlite3_vtab_cursor *cur){
+ ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+ int rc = SQLITE_OK;
+
+ if( pCsr->pFile ){
+ i64 iEof = pCsr->eocd.iOffset + pCsr->eocd.nSize;
+ zipfileEntryFree(pCsr->pCurrent);
+ pCsr->pCurrent = 0;
+ if( pCsr->iNextOff>=iEof ){
+ pCsr->bEof = 1;
+ }else{
+ ZipfileEntry *p = 0;
+ ZipfileTab *pTab = (ZipfileTab*)(cur->pVtab);
+ rc = zipfileGetEntry(pTab, 0, 0, pCsr->pFile, pCsr->iNextOff, &p);
+ if( rc==SQLITE_OK ){
+ pCsr->iNextOff += ZIPFILE_CDS_FIXED_SZ;
+ pCsr->iNextOff += (int)p->cds.nExtra + p->cds.nFile + p->cds.nComment;
+ }
+ pCsr->pCurrent = p;
+ }
+ }else{
+ if( !pCsr->bNoop ){
+ pCsr->pCurrent = pCsr->pCurrent->pNext;
+ }
+ if( pCsr->pCurrent==0 ){
+ pCsr->bEof = 1;
+ }
+ }
+
+ pCsr->bNoop = 0;
+ return rc;
+}
+
+static void zipfileFree(void *p) {
+ sqlite3_free(p);
+}
+
+/*
+** Buffer aIn (size nIn bytes) contains compressed data. Uncompressed, the
+** size is nOut bytes. This function uncompresses the data and sets the
+** return value in context pCtx to the result (a blob).
+**
+** If an error occurs, an error code is left in pCtx instead.
+*/
+static void zipfileInflate(
+ sqlite3_context *pCtx, /* Store result here */
+ const u8 *aIn, /* Compressed data */
+ int nIn, /* Size of buffer aIn[] in bytes */
+ int nOut /* Expected output size */
+){
+ u8 *aRes = sqlite3_malloc(nOut);
+ if( aRes==0 ){
+ sqlite3_result_error_nomem(pCtx);
+ }else{
+ int err;
+ z_stream str;
+ memset(&str, 0, sizeof(str));
+
+ str.next_in = (Byte*)aIn;
+ str.avail_in = nIn;
+ str.next_out = (Byte*)aRes;
+ str.avail_out = nOut;
+
+ err = inflateInit2(&str, -15);
+ if( err!=Z_OK ){
+ zipfileCtxErrorMsg(pCtx, "inflateInit2() failed (%d)", err);
+ }else{
+ err = inflate(&str, Z_NO_FLUSH);
+ if( err!=Z_STREAM_END ){
+ zipfileCtxErrorMsg(pCtx, "inflate() failed (%d)", err);
+ }else{
+ sqlite3_result_blob(pCtx, aRes, nOut, zipfileFree);
+ aRes = 0;
+ }
+ }
+ sqlite3_free(aRes);
+ inflateEnd(&str);
+ }
+}
+
+/*
+** Buffer aIn (size nIn bytes) contains uncompressed data. This function
+** compresses it and sets (*ppOut) to point to a buffer containing the
+** compressed data. The caller is responsible for eventually calling
+** sqlite3_free() to release buffer (*ppOut). Before returning, (*pnOut)
+** is set to the size of buffer (*ppOut) in bytes.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an SQLite error
+** code is returned and an error message left in virtual-table handle
+** pTab. The values of (*ppOut) and (*pnOut) are left unchanged in this
+** case.
+*/
+static int zipfileDeflate(
+ const u8 *aIn, int nIn, /* Input */
+ u8 **ppOut, int *pnOut, /* Output */
+ char **pzErr /* OUT: Error message */
+){
+ int nAlloc = (int)compressBound(nIn);
+ u8 *aOut;
+ int rc = SQLITE_OK;
+
+ aOut = (u8*)sqlite3_malloc(nAlloc);
+ if( aOut==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ int res;
+ z_stream str;
+ memset(&str, 0, sizeof(str));
+ str.next_in = (Bytef*)aIn;
+ str.avail_in = nIn;
+ str.next_out = aOut;
+ str.avail_out = nAlloc;
+
+ deflateInit2(&str, 9, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY);
+ res = deflate(&str, Z_FINISH);
+
+ if( res==Z_STREAM_END ){
+ *ppOut = aOut;
+ *pnOut = (int)str.total_out;
+ }else{
+ sqlite3_free(aOut);
+ *pzErr = sqlite3_mprintf("zipfile: deflate() error");
+ rc = SQLITE_ERROR;
+ }
+ deflateEnd(&str);
+ }
+
+ return rc;
+}
+
+
+/*
+** Return values of columns for the row at which the series_cursor
+** is currently pointing.
+*/
+static int zipfileColumn(
+ sqlite3_vtab_cursor *cur, /* The cursor */
+ sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
+ int i /* Which column to return */
+){
+ ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+ ZipfileCDS *pCDS = &pCsr->pCurrent->cds;
+ int rc = SQLITE_OK;
+ switch( i ){
+ case 0: /* name */
+ sqlite3_result_text(ctx, pCDS->zFile, -1, SQLITE_TRANSIENT);
+ break;
+ case 1: /* mode */
+ /* TODO: Whether or not the following is correct surely depends on
+ ** the platform on which the archive was created. */
+ sqlite3_result_int(ctx, pCDS->iExternalAttr >> 16);
+ break;
+ case 2: { /* mtime */
+ sqlite3_result_int64(ctx, pCsr->pCurrent->mUnixTime);
+ break;
+ }
+ case 3: { /* sz */
+ if( sqlite3_vtab_nochange(ctx)==0 ){
+ sqlite3_result_int64(ctx, pCDS->szUncompressed);
+ }
+ break;
+ }
+ case 4: /* rawdata */
+ if( sqlite3_vtab_nochange(ctx) ) break;
+ case 5: { /* data */
+ if( i==4 || pCDS->iCompression==0 || pCDS->iCompression==8 ){
+ int sz = pCDS->szCompressed;
+ int szFinal = pCDS->szUncompressed;
+ if( szFinal>0 ){
+ u8 *aBuf;
+ u8 *aFree = 0;
+ if( pCsr->pCurrent->aData ){
+ aBuf = pCsr->pCurrent->aData;
+ }else{
+ aBuf = aFree = sqlite3_malloc(sz);
+ if( aBuf==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ FILE *pFile = pCsr->pFile;
+ if( pFile==0 ){
+ pFile = ((ZipfileTab*)(pCsr->base.pVtab))->pWriteFd;
+ }
+ rc = zipfileReadData(pFile, aBuf, sz, pCsr->pCurrent->iDataOff,
+ &pCsr->base.pVtab->zErrMsg
+ );
+ }
+ }
+ if( rc==SQLITE_OK ){
+ if( i==5 && pCDS->iCompression ){
+ zipfileInflate(ctx, aBuf, sz, szFinal);
+ }else{
+ sqlite3_result_blob(ctx, aBuf, sz, SQLITE_TRANSIENT);
+ }
+ }
+ sqlite3_free(aFree);
+ }else{
+ /* Figure out if this is a directory or a zero-sized file. Consider
+ ** it to be a directory either if the mode suggests so, or if
+ ** the final character in the name is '/'. */
+ u32 mode = pCDS->iExternalAttr >> 16;
+ if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){
+ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC);
+ }
+ }
+ }
+ break;
+ }
+ case 6: /* method */
+ sqlite3_result_int(ctx, pCDS->iCompression);
+ break;
+ default: /* z */
+ assert( i==7 );
+ sqlite3_result_int64(ctx, pCsr->iId);
+ break;
+ }
+
+ return rc;
+}
+
+/*
+** Return TRUE if the cursor is at EOF.
+*/
+static int zipfileEof(sqlite3_vtab_cursor *cur){
+ ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+ return pCsr->bEof;
+}
+
+/*
+** If aBlob is not NULL, then it points to a buffer nBlob bytes in size
+** containing an entire zip archive image. Or, if aBlob is NULL, then pFile
+** is guaranteed to be a file-handle open on a zip file.
+**
+** This function attempts to locate the EOCD record within the zip archive
+** and populate *pEOCD with the results of decoding it. SQLITE_OK is
+** returned if successful. Otherwise, an SQLite error code is returned and
+** an English language error message may be left in virtual-table pTab.
+*/
+static int zipfileReadEOCD(
+ ZipfileTab *pTab, /* Return errors here */
+ const u8 *aBlob, /* Pointer to in-memory file image */
+ int nBlob, /* Size of aBlob[] in bytes */
+ FILE *pFile, /* Read from this file if aBlob==0 */
+ ZipfileEOCD *pEOCD /* Object to populate */
+){
+ u8 *aRead = pTab->aBuffer; /* Temporary buffer */
+ int nRead; /* Bytes to read from file */
+ int rc = SQLITE_OK;
+
+ if( aBlob==0 ){
+ i64 iOff; /* Offset to read from */
+ i64 szFile; /* Total size of file in bytes */
+ fseek(pFile, 0, SEEK_END);
+ szFile = (i64)ftell(pFile);
+ if( szFile==0 ){
+ memset(pEOCD, 0, sizeof(ZipfileEOCD));
+ return SQLITE_OK;
+ }
+ nRead = (int)(MIN(szFile, ZIPFILE_BUFFER_SIZE));
+ iOff = szFile - nRead;
+ rc = zipfileReadData(pFile, aRead, nRead, iOff, &pTab->base.zErrMsg);
+ }else{
+ nRead = (int)(MIN(nBlob, ZIPFILE_BUFFER_SIZE));
+ aRead = (u8*)&aBlob[nBlob-nRead];
+ }
+
+ if( rc==SQLITE_OK ){
+ int i;
+
+ /* Scan backwards looking for the signature bytes */
+ for(i=nRead-20; i>=0; i--){
+ if( aRead[i]==0x50 && aRead[i+1]==0x4b
+ && aRead[i+2]==0x05 && aRead[i+3]==0x06
+ ){
+ break;
+ }
+ }
+ if( i<0 ){
+ pTab->base.zErrMsg = sqlite3_mprintf(
+ "cannot find end of central directory record"
+ );
+ return SQLITE_ERROR;
+ }
+
+ aRead += i+4;
+ pEOCD->iDisk = zipfileRead16(aRead);
+ pEOCD->iFirstDisk = zipfileRead16(aRead);
+ pEOCD->nEntry = zipfileRead16(aRead);
+ pEOCD->nEntryTotal = zipfileRead16(aRead);
+ pEOCD->nSize = zipfileRead32(aRead);
+ pEOCD->iOffset = zipfileRead32(aRead);
+ }
+
+ return rc;
+}
+
+/*
+** Add object pNew to the linked list that begins at ZipfileTab.pFirstEntry
+** and ends with pLastEntry. If argument pBefore is NULL, then pNew is added
+** to the end of the list. Otherwise, it is added to the list immediately
+** before pBefore (which is guaranteed to be a part of said list).
+*/
+static void zipfileAddEntry(
+ ZipfileTab *pTab,
+ ZipfileEntry *pBefore,
+ ZipfileEntry *pNew
+){
+ assert( (pTab->pFirstEntry==0)==(pTab->pLastEntry==0) );
+ assert( pNew->pNext==0 );
+ if( pBefore==0 ){
+ if( pTab->pFirstEntry==0 ){
+ pTab->pFirstEntry = pTab->pLastEntry = pNew;
+ }else{
+ assert( pTab->pLastEntry->pNext==0 );
+ pTab->pLastEntry->pNext = pNew;
+ pTab->pLastEntry = pNew;
+ }
+ }else{
+ ZipfileEntry **pp;
+ for(pp=&pTab->pFirstEntry; *pp!=pBefore; pp=&((*pp)->pNext));
+ pNew->pNext = pBefore;
+ *pp = pNew;
+ }
+}
+
+static int zipfileLoadDirectory(ZipfileTab *pTab, const u8 *aBlob, int nBlob){
+ ZipfileEOCD eocd;
+ int rc;
+ int i;
+ i64 iOff;
+
+ rc = zipfileReadEOCD(pTab, aBlob, nBlob, pTab->pWriteFd, &eocd);
+ iOff = eocd.iOffset;
+ for(i=0; rc==SQLITE_OK && ipWriteFd, iOff, &pNew);
+
+ if( rc==SQLITE_OK ){
+ zipfileAddEntry(pTab, 0, pNew);
+ iOff += ZIPFILE_CDS_FIXED_SZ;
+ iOff += (int)pNew->cds.nExtra + pNew->cds.nFile + pNew->cds.nComment;
+ }
+ }
+ return rc;
+}
+
+/*
+** xFilter callback.
+*/
+static int zipfileFilter(
+ sqlite3_vtab_cursor *cur,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ ZipfileTab *pTab = (ZipfileTab*)cur->pVtab;
+ ZipfileCsr *pCsr = (ZipfileCsr*)cur;
+ const char *zFile = 0; /* Zip file to scan */
+ int rc = SQLITE_OK; /* Return Code */
+ int bInMemory = 0; /* True for an in-memory zipfile */
+
+ zipfileResetCursor(pCsr);
+
+ if( pTab->zFile ){
+ zFile = pTab->zFile;
+ }else if( idxNum==0 ){
+ zipfileCursorErr(pCsr, "zipfile() function requires an argument");
+ return SQLITE_ERROR;
+ }else if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){
+ const u8 *aBlob = (const u8*)sqlite3_value_blob(argv[0]);
+ int nBlob = sqlite3_value_bytes(argv[0]);
+ assert( pTab->pFirstEntry==0 );
+ rc = zipfileLoadDirectory(pTab, aBlob, nBlob);
+ pCsr->pFreeEntry = pTab->pFirstEntry;
+ pTab->pFirstEntry = pTab->pLastEntry = 0;
+ if( rc!=SQLITE_OK ) return rc;
+ bInMemory = 1;
+ }else{
+ zFile = (const char*)sqlite3_value_text(argv[0]);
+ }
+
+ if( 0==pTab->pWriteFd && 0==bInMemory ){
+ pCsr->pFile = fopen(zFile, "rb");
+ if( pCsr->pFile==0 ){
+ zipfileCursorErr(pCsr, "cannot open file: %s", zFile);
+ rc = SQLITE_ERROR;
+ }else{
+ rc = zipfileReadEOCD(pTab, 0, 0, pCsr->pFile, &pCsr->eocd);
+ if( rc==SQLITE_OK ){
+ if( pCsr->eocd.nEntry==0 ){
+ pCsr->bEof = 1;
+ }else{
+ pCsr->iNextOff = pCsr->eocd.iOffset;
+ rc = zipfileNext(cur);
+ }
+ }
+ }
+ }else{
+ pCsr->bNoop = 1;
+ pCsr->pCurrent = pCsr->pFreeEntry ? pCsr->pFreeEntry : pTab->pFirstEntry;
+ rc = zipfileNext(cur);
+ }
+
+ return rc;
+}
+
+/*
+** xBestIndex callback.
+*/
+static int zipfileBestIndex(
+ sqlite3_vtab *tab,
+ sqlite3_index_info *pIdxInfo
+){
+ int i;
+ int idx = -1;
+ int unusable = 0;
+
+ for(i=0; inConstraint; i++){
+ const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
+ if( pCons->iColumn!=ZIPFILE_F_COLUMN_IDX ) continue;
+ if( pCons->usable==0 ){
+ unusable = 1;
+ }else if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ idx = i;
+ }
+ }
+ if( idx>=0 ){
+ pIdxInfo->aConstraintUsage[idx].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[idx].omit = 1;
+ pIdxInfo->estimatedCost = 1000.0;
+ pIdxInfo->idxNum = 1;
+ }else if( unusable ){
+ return SQLITE_CONSTRAINT;
+ }
+ return SQLITE_OK;
+}
+
+static ZipfileEntry *zipfileNewEntry(const char *zPath){
+ ZipfileEntry *pNew;
+ pNew = sqlite3_malloc(sizeof(ZipfileEntry));
+ if( pNew ){
+ memset(pNew, 0, sizeof(ZipfileEntry));
+ pNew->cds.zFile = sqlite3_mprintf("%s", zPath);
+ if( pNew->cds.zFile==0 ){
+ sqlite3_free(pNew);
+ pNew = 0;
+ }
+ }
+ return pNew;
+}
+
+static int zipfileSerializeLFH(ZipfileEntry *pEntry, u8 *aBuf){
+ ZipfileCDS *pCds = &pEntry->cds;
+ u8 *a = aBuf;
+
+ pCds->nExtra = 9;
+
+ /* Write the LFH itself */
+ zipfileWrite32(a, ZIPFILE_SIGNATURE_LFH);
+ zipfileWrite16(a, pCds->iVersionExtract);
+ zipfileWrite16(a, pCds->flags);
+ zipfileWrite16(a, pCds->iCompression);
+ zipfileWrite16(a, pCds->mTime);
+ zipfileWrite16(a, pCds->mDate);
+ zipfileWrite32(a, pCds->crc32);
+ zipfileWrite32(a, pCds->szCompressed);
+ zipfileWrite32(a, pCds->szUncompressed);
+ zipfileWrite16(a, (u16)pCds->nFile);
+ zipfileWrite16(a, pCds->nExtra);
+ assert( a==&aBuf[ZIPFILE_LFH_FIXED_SZ] );
+
+ /* Add the file name */
+ memcpy(a, pCds->zFile, (int)pCds->nFile);
+ a += (int)pCds->nFile;
+
+ /* The "extra" data */
+ zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP);
+ zipfileWrite16(a, 5);
+ *a++ = 0x01;
+ zipfileWrite32(a, pEntry->mUnixTime);
+
+ return a-aBuf;
+}
+
+static int zipfileAppendEntry(
+ ZipfileTab *pTab,
+ ZipfileEntry *pEntry,
+ const u8 *pData,
+ int nData
+){
+ u8 *aBuf = pTab->aBuffer;
+ int nBuf;
+ int rc;
+
+ nBuf = zipfileSerializeLFH(pEntry, aBuf);
+ rc = zipfileAppendData(pTab, aBuf, nBuf);
+ if( rc==SQLITE_OK ){
+ pEntry->iDataOff = pTab->szCurrent;
+ rc = zipfileAppendData(pTab, pData, nData);
+ }
+
+ return rc;
+}
+
+static int zipfileGetMode(
+ sqlite3_value *pVal,
+ int bIsDir, /* If true, default to directory */
+ u32 *pMode, /* OUT: Mode value */
+ char **pzErr /* OUT: Error message */
+){
+ const char *z = (const char*)sqlite3_value_text(pVal);
+ u32 mode = 0;
+ if( z==0 ){
+ mode = (bIsDir ? (S_IFDIR + 0755) : (S_IFREG + 0644));
+ }else if( z[0]>='0' && z[0]<='9' ){
+ mode = (unsigned int)sqlite3_value_int(pVal);
+ }else{
+ const char zTemplate[11] = "-rwxrwxrwx";
+ int i;
+ if( strlen(z)!=10 ) goto parse_error;
+ switch( z[0] ){
+ case '-': mode |= S_IFREG; break;
+ case 'd': mode |= S_IFDIR; break;
+ case 'l': mode |= S_IFLNK; break;
+ default: goto parse_error;
+ }
+ for(i=1; i<10; i++){
+ if( z[i]==zTemplate[i] ) mode |= 1 << (9-i);
+ else if( z[i]!='-' ) goto parse_error;
+ }
+ }
+ if( ((mode & S_IFDIR)==0)==bIsDir ){
+ /* The "mode" attribute is a directory, but data has been specified.
+ ** Or vice-versa - no data but "mode" is a file or symlink. */
+ *pzErr = sqlite3_mprintf("zipfile: mode does not match data");
+ return SQLITE_CONSTRAINT;
+ }
+ *pMode = mode;
+ return SQLITE_OK;
+
+ parse_error:
+ *pzErr = sqlite3_mprintf("zipfile: parse error in mode: %s", z);
+ return SQLITE_ERROR;
+}
+
+/*
+** Both (const char*) arguments point to nul-terminated strings. Argument
+** nB is the value of strlen(zB). This function returns 0 if the strings are
+** identical, ignoring any trailing '/' character in either path. */
+static int zipfileComparePath(const char *zA, const char *zB, int nB){
+ int nA = (int)strlen(zA);
+ if( zA[nA-1]=='/' ) nA--;
+ if( zB[nB-1]=='/' ) nB--;
+ if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0;
+ return 1;
+}
+
+static int zipfileBegin(sqlite3_vtab *pVtab){
+ ZipfileTab *pTab = (ZipfileTab*)pVtab;
+ int rc = SQLITE_OK;
+
+ assert( pTab->pWriteFd==0 );
+
+ /* Open a write fd on the file. Also load the entire central directory
+ ** structure into memory. During the transaction any new file data is
+ ** appended to the archive file, but the central directory is accumulated
+ ** in main-memory until the transaction is committed. */
+ pTab->pWriteFd = fopen(pTab->zFile, "ab+");
+ if( pTab->pWriteFd==0 ){
+ pTab->base.zErrMsg = sqlite3_mprintf(
+ "zipfile: failed to open file %s for writing", pTab->zFile
+ );
+ rc = SQLITE_ERROR;
+ }else{
+ fseek(pTab->pWriteFd, 0, SEEK_END);
+ pTab->szCurrent = pTab->szOrig = (i64)ftell(pTab->pWriteFd);
+ rc = zipfileLoadDirectory(pTab, 0, 0);
+ }
+
+ if( rc!=SQLITE_OK ){
+ zipfileCleanupTransaction(pTab);
+ }
+
+ return rc;
+}
+
+/*
+** Return the current time as a 32-bit timestamp in UNIX epoch format (like
+** time(2)).
+*/
+static u32 zipfileTime(void){
+ sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
+ u32 ret;
+ if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){
+ i64 ms;
+ pVfs->xCurrentTimeInt64(pVfs, &ms);
+ ret = (u32)((ms/1000) - ((i64)24405875 * 8640));
+ }else{
+ double day;
+ pVfs->xCurrentTime(pVfs, &day);
+ ret = (u32)((day - 2440587.5) * 86400);
+ }
+ return ret;
+}
+
+/*
+** Return a 32-bit timestamp in UNIX epoch format.
+**
+** If the value passed as the only argument is either NULL or an SQL NULL,
+** return the current time. Otherwise, return the value stored in (*pVal)
+** cast to a 32-bit unsigned integer.
+*/
+static u32 zipfileGetTime(sqlite3_value *pVal){
+ if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
+ return zipfileTime();
+ }
+ return (u32)sqlite3_value_int64(pVal);
+}
+
+/*
+** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry
+** linked list. Remove it from the list and free the object.
+*/
+static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){
+ if( pOld ){
+ ZipfileEntry **pp;
+ for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
+ *pp = (*pp)->pNext;
+ zipfileEntryFree(pOld);
+ }
+}
+
+/*
+** xUpdate method.
+*/
+static int zipfileUpdate(
+ sqlite3_vtab *pVtab,
+ int nVal,
+ sqlite3_value **apVal,
+ sqlite_int64 *pRowid
+){
+ ZipfileTab *pTab = (ZipfileTab*)pVtab;
+ int rc = SQLITE_OK; /* Return Code */
+ ZipfileEntry *pNew = 0; /* New in-memory CDS entry */
+
+ u32 mode = 0; /* Mode for new entry */
+ u32 mTime = 0; /* Modification time for new entry */
+ i64 sz = 0; /* Uncompressed size */
+ const char *zPath = 0; /* Path for new entry */
+ int nPath = 0; /* strlen(zPath) */
+ const u8 *pData = 0; /* Pointer to buffer containing content */
+ int nData = 0; /* Size of pData buffer in bytes */
+ int iMethod = 0; /* Compression method for new entry */
+ u8 *pFree = 0; /* Free this */
+ char *zFree = 0; /* Also free this */
+ ZipfileEntry *pOld = 0;
+ ZipfileEntry *pOld2 = 0;
+ int bUpdate = 0; /* True for an update that modifies "name" */
+ int bIsDir = 0;
+ u32 iCrc32 = 0;
+
+ if( pTab->pWriteFd==0 ){
+ rc = zipfileBegin(pVtab);
+ if( rc!=SQLITE_OK ) return rc;
+ }
+
+ /* If this is a DELETE or UPDATE, find the archive entry to delete. */
+ if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
+ const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
+ int nDelete = (int)strlen(zDelete);
+ if( nVal>1 ){
+ const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]);
+ if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){
+ bUpdate = 1;
+ }
+ }
+ for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
+ if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){
+ break;
+ }
+ assert( pOld->pNext );
+ }
+ }
+
+ if( nVal>1 ){
+ /* Check that "sz" and "rawdata" are both NULL: */
+ if( sqlite3_value_type(apVal[5])!=SQLITE_NULL ){
+ zipfileTableErr(pTab, "sz must be NULL");
+ rc = SQLITE_CONSTRAINT;
+ }
+ if( sqlite3_value_type(apVal[6])!=SQLITE_NULL ){
+ zipfileTableErr(pTab, "rawdata must be NULL");
+ rc = SQLITE_CONSTRAINT;
+ }
+
+ if( rc==SQLITE_OK ){
+ if( sqlite3_value_type(apVal[7])==SQLITE_NULL ){
+ /* data=NULL. A directory */
+ bIsDir = 1;
+ }else{
+ /* Value specified for "data", and possibly "method". This must be
+ ** a regular file or a symlink. */
+ const u8 *aIn = sqlite3_value_blob(apVal[7]);
+ int nIn = sqlite3_value_bytes(apVal[7]);
+ int bAuto = sqlite3_value_type(apVal[8])==SQLITE_NULL;
+
+ iMethod = sqlite3_value_int(apVal[8]);
+ sz = nIn;
+ pData = aIn;
+ nData = nIn;
+ if( iMethod!=0 && iMethod!=8 ){
+ zipfileTableErr(pTab, "unknown compression method: %d", iMethod);
+ rc = SQLITE_CONSTRAINT;
+ }else{
+ if( bAuto || iMethod ){
+ int nCmp;
+ rc = zipfileDeflate(aIn, nIn, &pFree, &nCmp, &pTab->base.zErrMsg);
+ if( rc==SQLITE_OK ){
+ if( iMethod || nCmpbase.zErrMsg);
+ }
+
+ if( rc==SQLITE_OK ){
+ zPath = (const char*)sqlite3_value_text(apVal[2]);
+ nPath = (int)strlen(zPath);
+ mTime = zipfileGetTime(apVal[4]);
+ }
+
+ if( rc==SQLITE_OK && bIsDir ){
+ /* For a directory, check that the last character in the path is a
+ ** '/'. This appears to be required for compatibility with info-zip
+ ** (the unzip command on unix). It does not create directories
+ ** otherwise. */
+ if( zPath[nPath-1]!='/' ){
+ zFree = sqlite3_mprintf("%s/", zPath);
+ if( zFree==0 ){ rc = SQLITE_NOMEM; }
+ zPath = (const char*)zFree;
+ nPath++;
+ }
+ }
+
+ /* Check that we're not inserting a duplicate entry -OR- updating an
+ ** entry with a path, thereby making it into a duplicate. */
+ if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){
+ ZipfileEntry *p;
+ for(p=pTab->pFirstEntry; p; p=p->pNext){
+ if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){
+ switch( sqlite3_vtab_on_conflict(pTab->db) ){
+ case SQLITE_IGNORE: {
+ goto zipfile_update_done;
+ }
+ case SQLITE_REPLACE: {
+ pOld2 = p;
+ break;
+ }
+ default: {
+ zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath);
+ rc = SQLITE_CONSTRAINT;
+ break;
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ /* Create the new CDS record. */
+ pNew = zipfileNewEntry(zPath);
+ if( pNew==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ pNew->cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+ pNew->cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+ pNew->cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+ pNew->cds.iCompression = (u16)iMethod;
+ zipfileMtimeToDos(&pNew->cds, mTime);
+ pNew->cds.crc32 = iCrc32;
+ pNew->cds.szCompressed = nData;
+ pNew->cds.szUncompressed = (u32)sz;
+ pNew->cds.iExternalAttr = (mode<<16);
+ pNew->cds.iOffset = (u32)pTab->szCurrent;
+ pNew->cds.nFile = (u16)nPath;
+ pNew->mUnixTime = (u32)mTime;
+ rc = zipfileAppendEntry(pTab, pNew, pData, nData);
+ zipfileAddEntry(pTab, pOld, pNew);
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK && (pOld || pOld2) ){
+ ZipfileCsr *pCsr;
+ for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
+ if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){
+ pCsr->pCurrent = pCsr->pCurrent->pNext;
+ pCsr->bNoop = 1;
+ }
+ }
+
+ zipfileRemoveEntryFromList(pTab, pOld);
+ zipfileRemoveEntryFromList(pTab, pOld2);
+ }
+
+zipfile_update_done:
+ sqlite3_free(pFree);
+ sqlite3_free(zFree);
+ return rc;
+}
+
+static int zipfileSerializeEOCD(ZipfileEOCD *p, u8 *aBuf){
+ u8 *a = aBuf;
+ zipfileWrite32(a, ZIPFILE_SIGNATURE_EOCD);
+ zipfileWrite16(a, p->iDisk);
+ zipfileWrite16(a, p->iFirstDisk);
+ zipfileWrite16(a, p->nEntry);
+ zipfileWrite16(a, p->nEntryTotal);
+ zipfileWrite32(a, p->nSize);
+ zipfileWrite32(a, p->iOffset);
+ zipfileWrite16(a, 0); /* Size of trailing comment in bytes*/
+
+ return a-aBuf;
+}
+
+static int zipfileAppendEOCD(ZipfileTab *pTab, ZipfileEOCD *p){
+ int nBuf = zipfileSerializeEOCD(p, pTab->aBuffer);
+ assert( nBuf==ZIPFILE_EOCD_FIXED_SZ );
+ return zipfileAppendData(pTab, pTab->aBuffer, nBuf);
+}
+
+/*
+** Serialize the CDS structure into buffer aBuf[]. Return the number
+** of bytes written.
+*/
+static int zipfileSerializeCDS(ZipfileEntry *pEntry, u8 *aBuf){
+ u8 *a = aBuf;
+ ZipfileCDS *pCDS = &pEntry->cds;
+
+ if( pEntry->aExtra==0 ){
+ pCDS->nExtra = 9;
+ }
+
+ zipfileWrite32(a, ZIPFILE_SIGNATURE_CDS);
+ zipfileWrite16(a, pCDS->iVersionMadeBy);
+ zipfileWrite16(a, pCDS->iVersionExtract);
+ zipfileWrite16(a, pCDS->flags);
+ zipfileWrite16(a, pCDS->iCompression);
+ zipfileWrite16(a, pCDS->mTime);
+ zipfileWrite16(a, pCDS->mDate);
+ zipfileWrite32(a, pCDS->crc32);
+ zipfileWrite32(a, pCDS->szCompressed);
+ zipfileWrite32(a, pCDS->szUncompressed);
+ assert( a==&aBuf[ZIPFILE_CDS_NFILE_OFF] );
+ zipfileWrite16(a, pCDS->nFile);
+ zipfileWrite16(a, pCDS->nExtra);
+ zipfileWrite16(a, pCDS->nComment);
+ zipfileWrite16(a, pCDS->iDiskStart);
+ zipfileWrite16(a, pCDS->iInternalAttr);
+ zipfileWrite32(a, pCDS->iExternalAttr);
+ zipfileWrite32(a, pCDS->iOffset);
+
+ memcpy(a, pCDS->zFile, pCDS->nFile);
+ a += pCDS->nFile;
+
+ if( pEntry->aExtra ){
+ int n = (int)pCDS->nExtra + (int)pCDS->nComment;
+ memcpy(a, pEntry->aExtra, n);
+ a += n;
+ }else{
+ assert( pCDS->nExtra==9 );
+ zipfileWrite16(a, ZIPFILE_EXTRA_TIMESTAMP);
+ zipfileWrite16(a, 5);
+ *a++ = 0x01;
+ zipfileWrite32(a, pEntry->mUnixTime);
+ }
+
+ return a-aBuf;
+}
+
+static int zipfileCommit(sqlite3_vtab *pVtab){
+ ZipfileTab *pTab = (ZipfileTab*)pVtab;
+ int rc = SQLITE_OK;
+ if( pTab->pWriteFd ){
+ i64 iOffset = pTab->szCurrent;
+ ZipfileEntry *p;
+ ZipfileEOCD eocd;
+ int nEntry = 0;
+
+ /* Write out all entries */
+ for(p=pTab->pFirstEntry; rc==SQLITE_OK && p; p=p->pNext){
+ int n = zipfileSerializeCDS(p, pTab->aBuffer);
+ rc = zipfileAppendData(pTab, pTab->aBuffer, n);
+ nEntry++;
+ }
+
+ /* Write out the EOCD record */
+ eocd.iDisk = 0;
+ eocd.iFirstDisk = 0;
+ eocd.nEntry = (u16)nEntry;
+ eocd.nEntryTotal = (u16)nEntry;
+ eocd.nSize = (u32)(pTab->szCurrent - iOffset);
+ eocd.iOffset = (u32)iOffset;
+ rc = zipfileAppendEOCD(pTab, &eocd);
+
+ zipfileCleanupTransaction(pTab);
+ }
+ return rc;
+}
+
+static int zipfileRollback(sqlite3_vtab *pVtab){
+ return zipfileCommit(pVtab);
+}
+
+static ZipfileCsr *zipfileFindCursor(ZipfileTab *pTab, i64 iId){
+ ZipfileCsr *pCsr;
+ for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
+ if( iId==pCsr->iId ) break;
+ }
+ return pCsr;
+}
+
+static void zipfileFunctionCds(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ ZipfileCsr *pCsr;
+ ZipfileTab *pTab = (ZipfileTab*)sqlite3_user_data(context);
+ assert( argc>0 );
+
+ pCsr = zipfileFindCursor(pTab, sqlite3_value_int64(argv[0]));
+ if( pCsr ){
+ ZipfileCDS *p = &pCsr->pCurrent->cds;
+ char *zRes = sqlite3_mprintf("{"
+ "\"version-made-by\" : %u, "
+ "\"version-to-extract\" : %u, "
+ "\"flags\" : %u, "
+ "\"compression\" : %u, "
+ "\"time\" : %u, "
+ "\"date\" : %u, "
+ "\"crc32\" : %u, "
+ "\"compressed-size\" : %u, "
+ "\"uncompressed-size\" : %u, "
+ "\"file-name-length\" : %u, "
+ "\"extra-field-length\" : %u, "
+ "\"file-comment-length\" : %u, "
+ "\"disk-number-start\" : %u, "
+ "\"internal-attr\" : %u, "
+ "\"external-attr\" : %u, "
+ "\"offset\" : %u }",
+ (u32)p->iVersionMadeBy, (u32)p->iVersionExtract,
+ (u32)p->flags, (u32)p->iCompression,
+ (u32)p->mTime, (u32)p->mDate,
+ (u32)p->crc32, (u32)p->szCompressed,
+ (u32)p->szUncompressed, (u32)p->nFile,
+ (u32)p->nExtra, (u32)p->nComment,
+ (u32)p->iDiskStart, (u32)p->iInternalAttr,
+ (u32)p->iExternalAttr, (u32)p->iOffset
+ );
+
+ if( zRes==0 ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_text(context, zRes, -1, SQLITE_TRANSIENT);
+ sqlite3_free(zRes);
+ }
+ }
+}
+
+/*
+** xFindFunction method.
+*/
+static int zipfileFindFunction(
+ sqlite3_vtab *pVtab, /* Virtual table handle */
+ int nArg, /* Number of SQL function arguments */
+ const char *zName, /* Name of SQL function */
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
+ void **ppArg /* OUT: User data for *pxFunc */
+){
+ if( sqlite3_stricmp("zipfile_cds", zName)==0 ){
+ *pxFunc = zipfileFunctionCds;
+ *ppArg = (void*)pVtab;
+ return 1;
+ }
+ return 0;
+}
+
+typedef struct ZipfileBuffer ZipfileBuffer;
+struct ZipfileBuffer {
+ u8 *a; /* Pointer to buffer */
+ int n; /* Size of buffer in bytes */
+ int nAlloc; /* Byte allocated at a[] */
+};
+
+typedef struct ZipfileCtx ZipfileCtx;
+struct ZipfileCtx {
+ int nEntry;
+ ZipfileBuffer body;
+ ZipfileBuffer cds;
+};
+
+static int zipfileBufferGrow(ZipfileBuffer *pBuf, int nByte){
+ if( pBuf->n+nByte>pBuf->nAlloc ){
+ u8 *aNew;
+ int nNew = pBuf->n ? pBuf->n*2 : 512;
+ int nReq = pBuf->n + nByte;
+
+ while( nNewa, nNew);
+ if( aNew==0 ) return SQLITE_NOMEM;
+ pBuf->a = aNew;
+ pBuf->nAlloc = nNew;
+ }
+ return SQLITE_OK;
+}
+
+/*
+** xStep() callback for the zipfile() aggregate. This can be called in
+** any of the following ways:
+**
+** SELECT zipfile(name,data) ...
+** SELECT zipfile(name,mode,mtime,data) ...
+** SELECT zipfile(name,mode,mtime,data,method) ...
+*/
+void zipfileStep(sqlite3_context *pCtx, int nVal, sqlite3_value **apVal){
+ ZipfileCtx *p; /* Aggregate function context */
+ ZipfileEntry e; /* New entry to add to zip archive */
+
+ sqlite3_value *pName = 0;
+ sqlite3_value *pMode = 0;
+ sqlite3_value *pMtime = 0;
+ sqlite3_value *pData = 0;
+ sqlite3_value *pMethod = 0;
+
+ int bIsDir = 0;
+ u32 mode;
+ int rc = SQLITE_OK;
+ char *zErr = 0;
+
+ int iMethod = -1; /* Compression method to use (0 or 8) */
+
+ const u8 *aData = 0; /* Possibly compressed data for new entry */
+ int nData = 0; /* Size of aData[] in bytes */
+ int szUncompressed = 0; /* Size of data before compression */
+ u8 *aFree = 0; /* Free this before returning */
+ u32 iCrc32 = 0; /* crc32 of uncompressed data */
+
+ char *zName = 0; /* Path (name) of new entry */
+ int nName = 0; /* Size of zName in bytes */
+ char *zFree = 0; /* Free this before returning */
+ int nByte;
+
+ memset(&e, 0, sizeof(e));
+ p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx));
+ if( p==0 ) return;
+
+ /* Martial the arguments into stack variables */
+ if( nVal!=2 && nVal!=4 && nVal!=5 ){
+ zErr = sqlite3_mprintf("wrong number of arguments to function zipfile()");
+ rc = SQLITE_ERROR;
+ goto zipfile_step_out;
+ }
+ pName = apVal[0];
+ if( nVal==2 ){
+ pData = apVal[1];
+ }else{
+ pMode = apVal[1];
+ pMtime = apVal[2];
+ pData = apVal[3];
+ if( nVal==5 ){
+ pMethod = apVal[4];
+ }
+ }
+
+ /* Check that the 'name' parameter looks ok. */
+ zName = (char*)sqlite3_value_text(pName);
+ nName = sqlite3_value_bytes(pName);
+ if( zName==0 ){
+ zErr = sqlite3_mprintf("first argument to zipfile() must be non-NULL");
+ rc = SQLITE_ERROR;
+ goto zipfile_step_out;
+ }
+
+ /* Inspect the 'method' parameter. This must be either 0 (store), 8 (use
+ ** deflate compression) or NULL (choose automatically). */
+ if( pMethod && SQLITE_NULL!=sqlite3_value_type(pMethod) ){
+ iMethod = (int)sqlite3_value_int64(pMethod);
+ if( iMethod!=0 && iMethod!=8 ){
+ zErr = sqlite3_mprintf("illegal method value: %d", iMethod);
+ rc = SQLITE_ERROR;
+ goto zipfile_step_out;
+ }
+ }
+
+ /* Now inspect the data. If this is NULL, then the new entry must be a
+ ** directory. Otherwise, figure out whether or not the data should
+ ** be deflated or simply stored in the zip archive. */
+ if( sqlite3_value_type(pData)==SQLITE_NULL ){
+ bIsDir = 1;
+ iMethod = 0;
+ }else{
+ aData = sqlite3_value_blob(pData);
+ szUncompressed = nData = sqlite3_value_bytes(pData);
+ iCrc32 = crc32(0, aData, nData);
+ if( iMethod<0 || iMethod==8 ){
+ int nOut = 0;
+ rc = zipfileDeflate(aData, nData, &aFree, &nOut, &zErr);
+ if( rc!=SQLITE_OK ){
+ goto zipfile_step_out;
+ }
+ if( iMethod==8 || nOut1 && zName[nName-2]=='/' ) nName--;
+ }
+ }
+
+ /* Assemble the ZipfileEntry object for the new zip archive entry */
+ e.cds.iVersionMadeBy = ZIPFILE_NEWENTRY_MADEBY;
+ e.cds.iVersionExtract = ZIPFILE_NEWENTRY_REQUIRED;
+ e.cds.flags = ZIPFILE_NEWENTRY_FLAGS;
+ e.cds.iCompression = (u16)iMethod;
+ zipfileMtimeToDos(&e.cds, (u32)e.mUnixTime);
+ e.cds.crc32 = iCrc32;
+ e.cds.szCompressed = nData;
+ e.cds.szUncompressed = szUncompressed;
+ e.cds.iExternalAttr = (mode<<16);
+ e.cds.iOffset = p->body.n;
+ e.cds.nFile = (u16)nName;
+ e.cds.zFile = zName;
+
+ /* Append the LFH to the body of the new archive */
+ nByte = ZIPFILE_LFH_FIXED_SZ + e.cds.nFile + 9;
+ if( (rc = zipfileBufferGrow(&p->body, nByte)) ) goto zipfile_step_out;
+ p->body.n += zipfileSerializeLFH(&e, &p->body.a[p->body.n]);
+
+ /* Append the data to the body of the new archive */
+ if( nData>0 ){
+ if( (rc = zipfileBufferGrow(&p->body, nData)) ) goto zipfile_step_out;
+ memcpy(&p->body.a[p->body.n], aData, nData);
+ p->body.n += nData;
+ }
+
+ /* Append the CDS record to the directory of the new archive */
+ nByte = ZIPFILE_CDS_FIXED_SZ + e.cds.nFile + 9;
+ if( (rc = zipfileBufferGrow(&p->cds, nByte)) ) goto zipfile_step_out;
+ p->cds.n += zipfileSerializeCDS(&e, &p->cds.a[p->cds.n]);
+
+ /* Increment the count of entries in the archive */
+ p->nEntry++;
+
+ zipfile_step_out:
+ sqlite3_free(aFree);
+ sqlite3_free(zFree);
+ if( rc ){
+ if( zErr ){
+ sqlite3_result_error(pCtx, zErr, -1);
+ }else{
+ sqlite3_result_error_code(pCtx, rc);
+ }
+ }
+ sqlite3_free(zErr);
+}
+
+/*
+** xFinalize() callback for zipfile aggregate function.
+*/
+void zipfileFinal(sqlite3_context *pCtx){
+ ZipfileCtx *p;
+ ZipfileEOCD eocd;
+ int nZip;
+ u8 *aZip;
+
+ p = (ZipfileCtx*)sqlite3_aggregate_context(pCtx, sizeof(ZipfileCtx));
+ if( p==0 ) return;
+ if( p->nEntry>0 ){
+ memset(&eocd, 0, sizeof(eocd));
+ eocd.nEntry = (u16)p->nEntry;
+ eocd.nEntryTotal = (u16)p->nEntry;
+ eocd.nSize = p->cds.n;
+ eocd.iOffset = p->body.n;
+
+ nZip = p->body.n + p->cds.n + ZIPFILE_EOCD_FIXED_SZ;
+ aZip = (u8*)sqlite3_malloc(nZip);
+ if( aZip==0 ){
+ sqlite3_result_error_nomem(pCtx);
+ }else{
+ memcpy(aZip, p->body.a, p->body.n);
+ memcpy(&aZip[p->body.n], p->cds.a, p->cds.n);
+ zipfileSerializeEOCD(&eocd, &aZip[p->body.n + p->cds.n]);
+ sqlite3_result_blob(pCtx, aZip, nZip, zipfileFree);
+ }
+ }
+
+ sqlite3_free(p->body.a);
+ sqlite3_free(p->cds.a);
+}
+
+
+/*
+** Register the "zipfile" virtual table.
+*/
+static int zipfileRegister(sqlite3 *db){
+ static sqlite3_module zipfileModule = {
+ 1, /* iVersion */
+ zipfileConnect, /* xCreate */
+ zipfileConnect, /* xConnect */
+ zipfileBestIndex, /* xBestIndex */
+ zipfileDisconnect, /* xDisconnect */
+ zipfileDisconnect, /* xDestroy */
+ zipfileOpen, /* xOpen - open a cursor */
+ zipfileClose, /* xClose - close a cursor */
+ zipfileFilter, /* xFilter - configure scan constraints */
+ zipfileNext, /* xNext - advance a cursor */
+ zipfileEof, /* xEof - check for end of scan */
+ zipfileColumn, /* xColumn - read data */
+ 0, /* xRowid - read data */
+ zipfileUpdate, /* xUpdate */
+ zipfileBegin, /* xBegin */
+ 0, /* xSync */
+ zipfileCommit, /* xCommit */
+ zipfileRollback, /* xRollback */
+ zipfileFindFunction, /* xFindMethod */
+ 0, /* xRename */
+ };
+
+ int rc = sqlite3_create_module(db, "zipfile" , &zipfileModule, 0);
+ if( rc==SQLITE_OK ) rc = sqlite3_overload_function(db, "zipfile_cds", -1);
+ if( rc==SQLITE_OK ){
+ rc = sqlite3_create_function(db, "zipfile", -1, SQLITE_UTF8, 0, 0,
+ zipfileStep, zipfileFinal
+ );
+ }
+ return rc;
+}
+#else /* SQLITE_OMIT_VIRTUALTABLE */
+# define zipfileRegister(x) SQLITE_OK
+#endif
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_zipfile_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi);
+ (void)pzErrMsg; /* Unused parameter */
+ return zipfileRegister(db);
+}
ADDED ext/misc/zorder.c
Index: ext/misc/zorder.c
==================================================================
--- /dev/null
+++ ext/misc/zorder.c
@@ -0,0 +1,102 @@
+/*
+** 2018-02-09
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** SQL functions for z-order (Morton code) transformations.
+**
+** zorder(X0,X0,..,xN) Generate an N+1 dimension Morton code
+**
+** unzorder(Z,N,I) Extract the I-th dimension from N-dimensional
+** Morton code Z.
+*/
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+#include
+#include
+
+/*
+** Functions: zorder(X0,X1,....)
+**
+** Convert integers X0, X1, ... into morton code.
+**
+** The output is a signed 64-bit integer. If any argument is too large,
+** an error is thrown.
+*/
+static void zorderFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ sqlite3_int64 z, x[63];
+ int i, j;
+ z = 0;
+ for(i=0; i0 ){
+ for(i=0; i<63; i++){
+ j = i%argc;
+ z |= (x[j]&1)<>= 1;
+ }
+ }
+ sqlite3_result_int64(context, z);
+ for(i=0; i>j)&1)<1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){
bVacuum = 1;
+ }else if( nArg>1 && nArg<=7
+ && 0==memcmp(zArg, "-presql", nArg) && i1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i1 && nArg<=9
+ && 0==memcmp(zArg, "-statstep", nArg) && i0 && (i % nStatStep)==0 ){
+ sqlite3_int64 nUsed;
+ sqlite3_int64 nHighwater;
+ sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0);
+ fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater);
+ if( bVacuum==0 ){
+ int one;
+ int two;
+ sqlite3rbu_bp_progress(pRbu, &one, &two);
+ fprintf(stdout, " progress=%d/%d\n", one, two);
+ }else{
+ fprintf(stdout, "\n");
+ }
+ fflush(stdout);
+ }
+ }
+ nProgress = sqlite3rbu_progress(pRbu);
+ rc = sqlite3rbu_close(pRbu, &zErrmsg);
+ }
/* Let the user know what happened. */
switch( rc ){
case SQLITE_OK:
sqlite3_snprintf(sizeof(zBuf), zBuf,
Index: ext/rbu/rbu1.test
==================================================================
--- ext/rbu/rbu1.test
+++ ext/rbu/rbu1.test
@@ -137,10 +137,11 @@
eval $create_vfs
foreach {tn2 cmd} {
1 run_rbu
2 step_rbu 3 step_rbu_uri 4 step_rbu_state
+ 5 step_rbu_legacy
} {
foreach {tn schema} {
1 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
@@ -667,6 +668,5 @@
eval $destroy_vfs
}
finish_test
-
Index: ext/rbu/rbu10.test
==================================================================
--- ext/rbu/rbu10.test
+++ ext/rbu/rbu10.test
@@ -183,6 +183,5 @@
list [catch { apply_rbu $rbu } msg] $msg
} {0 SQLITE_DONE}
finish_test
-
Index: ext/rbu/rbu11.test
==================================================================
--- ext/rbu/rbu11.test
+++ ext/rbu/rbu11.test
@@ -193,6 +193,5 @@
do_test 4.7.2 {
list [catch {rbu close} msg] $msg
} {1 {SQLITE_ERROR - rbu_state mismatch error}}
finish_test
-
Index: ext/rbu/rbu12.test
==================================================================
--- ext/rbu/rbu12.test
+++ ext/rbu/rbu12.test
@@ -230,6 +230,5 @@
} [list $V1 $V2]
}
finish_test
-
Index: ext/rbu/rbu13.test
==================================================================
--- ext/rbu/rbu13.test
+++ ext/rbu/rbu13.test
@@ -60,6 +60,5 @@
a == ( (b<<6) + (c<<5) + (d<<4) + (e<<3) + (f<<2) + (g<<1) + (h<<0) )
} {128}
finish_test
-
Index: ext/rbu/rbu14.test
==================================================================
--- ext/rbu/rbu14.test
+++ ext/rbu/rbu14.test
@@ -90,6 +90,5 @@
integrity_check $tn.4
}
finish_test
-
Index: ext/rbu/rbu3.test
==================================================================
--- ext/rbu/rbu3.test
+++ ext/rbu/rbu3.test
@@ -201,7 +201,5 @@
} 1
do_test 6.1 { sqlite3rbu_internal_test } {}
finish_test
-
-
Index: ext/rbu/rbu5.test
==================================================================
--- ext/rbu/rbu5.test
+++ ext/rbu/rbu5.test
@@ -298,9 +298,5 @@
}
}
finish_test
-
-
-
-
Index: ext/rbu/rbu6.test
==================================================================
--- ext/rbu/rbu6.test
+++ ext/rbu/rbu6.test
@@ -98,6 +98,5 @@
} {1 t1 5 hello}
}
finish_test
-
Index: ext/rbu/rbu7.test
==================================================================
--- ext/rbu/rbu7.test
+++ ext/rbu/rbu7.test
@@ -104,7 +104,5 @@
3 2 f
}
}
finish_test
-
-
Index: ext/rbu/rbu8.test
==================================================================
--- ext/rbu/rbu8.test
+++ ext/rbu/rbu8.test
@@ -70,6 +70,5 @@
}
integrity_check 1.3.3
finish_test
-
Index: ext/rbu/rbu9.test
==================================================================
--- ext/rbu/rbu9.test
+++ ext/rbu/rbu9.test
@@ -123,6 +123,5 @@
integrity_check 2.$tn.4
}
finish_test
-
Index: ext/rbu/rbuA.test
==================================================================
--- ext/rbu/rbuA.test
+++ ext/rbu/rbuA.test
@@ -78,6 +78,5 @@
list [catch { rbu close } msg] $msg
} {1 {SQLITE_ERROR - cannot update wal mode database}}
finish_test
-
Index: ext/rbu/rbuB.test
==================================================================
--- ext/rbu/rbuB.test
+++ ext/rbu/rbuB.test
@@ -57,6 +57,5 @@
db close
sqlite3_shutdown
test_sqlite3_log
sqlite3_initialize
finish_test
-
Index: ext/rbu/rbuC.test
==================================================================
--- ext/rbu/rbuC.test
+++ ext/rbu/rbuC.test
@@ -137,6 +137,5 @@
}
finish_test
-
Index: ext/rbu/rbu_common.tcl
==================================================================
--- ext/rbu/rbu_common.tcl
+++ ext/rbu/rbu_common.tcl
@@ -67,12 +67,29 @@
rbu close
if {$rc != "SQLITE_OK"} break
}
set rc
}
+
+proc step_rbu_legacy {target rbu} {
+ while 1 {
+ sqlite3rbu rbu $target $rbu
+ set state [rbu state]
+ check_prestep_state $target $state
+ set rc [rbu step]
+ check_poststep_state $rc $target $state
+ rbu close
+ if {$rc != "SQLITE_OK"} break
+ sqlite3 tmpdb $rbu
+ tmpdb eval { DELETE FROM rbu_state WHERE k==10 }
+ tmpdb close
+ }
+ set rc
+}
proc do_rbu_vacuum_test {tn step} {
+ forcedelete state.db
uplevel [list do_test $tn.1 {
if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db }
while 1 {
if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db }
set state [rbu state]
ADDED ext/rbu/rbucollate.test
Index: ext/rbu/rbucollate.test
==================================================================
--- /dev/null
+++ ext/rbu/rbucollate.test
@@ -0,0 +1,62 @@
+# 2018 March 22
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbucollate
+
+ifcapable !icu_collations {
+ finish_test
+ return
+}
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+
+# Create a simple RBU database. That expects to write to a table:
+#
+# CREATE TABLE t1(a PRIMARY KEY, b, c);
+#
+proc create_rbu1 {filename} {
+ forcedelete $filename
+ sqlite3 rbu1 $filename
+ rbu1 eval {
+ CREATE TABLE data_t1(a, b, c, rbu_control);
+ INSERT INTO data_t1 VALUES('a', 'one', 1, 0);
+ INSERT INTO data_t1 VALUES('b', 'two', 2, 0);
+ INSERT INTO data_t1 VALUES('c', 'three', 3, 0);
+ }
+ rbu1 close
+ return $filename
+}
+
+do_execsql_test 1.0 {
+ SELECT icu_load_collation('en_US', 'my-collate');
+ CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c);
+} {{}}
+
+do_test 1.2 {
+ create_rbu1 testrbu.db
+ sqlite3rbu rbu test.db testrbu.db
+ rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+ rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') }
+ while 1 {
+ set rc [rbu step]
+ if {$rc!="SQLITE_OK"} break
+ }
+ rbu close
+ db eval { SELECT * FROM t1 }
+} {a one 1 b two 2 c three 3}
+
+#forcedelete testrbu.db
+finish_test
Index: ext/rbu/rbucrash.test
==================================================================
--- ext/rbu/rbucrash.test
+++ ext/rbu/rbucrash.test
@@ -143,6 +143,5 @@
do_rbu_crash_test 2.pre=$nPre.step=$is $nPre $is
}
}
finish_test
-
Index: ext/rbu/rbucrash2.test
==================================================================
--- ext/rbu/rbucrash2.test
+++ ext/rbu/rbucrash2.test
@@ -101,6 +101,5 @@
rbu close
}
}
finish_test
-
Index: ext/rbu/rbudiff.test
==================================================================
--- ext/rbu/rbudiff.test
+++ ext/rbu/rbudiff.test
@@ -298,6 +298,5 @@
}
}
finish_test
-
Index: ext/rbu/rbudor.test
==================================================================
--- ext/rbu/rbudor.test
+++ ext/rbu/rbudor.test
@@ -54,6 +54,5 @@
do_execsql_test 1.4 {
SELECT * FROM t1
} [list 1 $bigA 2 $bigB]
finish_test
-
Index: ext/rbu/rbufault.test
==================================================================
--- ext/rbu/rbufault.test
+++ ext/rbu/rbufault.test
@@ -232,6 +232,5 @@
}
}
}
finish_test
-
Index: ext/rbu/rbufault2.test
==================================================================
--- ext/rbu/rbufault2.test
+++ ext/rbu/rbufault2.test
@@ -53,6 +53,5 @@
finish_test
-
Index: ext/rbu/rbufault3.test
==================================================================
--- ext/rbu/rbufault3.test
+++ ext/rbu/rbufault3.test
@@ -93,6 +93,5 @@
}
}
finish_test
-
Index: ext/rbu/rbufault4.test
==================================================================
--- ext/rbu/rbufault4.test
+++ ext/rbu/rbufault4.test
@@ -61,6 +61,5 @@
}
finish_test
-
Index: ext/rbu/rbufts.test
==================================================================
--- ext/rbu/rbufts.test
+++ ext/rbu/rbufts.test
@@ -129,6 +129,5 @@
} {1 {SQLITE_ERROR - SQL logic error]}}
finish_test
-
ADDED ext/rbu/rbumulti.test
Index: ext/rbu/rbumulti.test
==================================================================
--- /dev/null
+++ ext/rbu/rbumulti.test
@@ -0,0 +1,174 @@
+# 2018 January 11
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests of multiple RBU operations running
+# concurrently within the same process.
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbumulti
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+autoinstall_test_functions
+
+proc build_db {db} {
+ $db eval {
+ CREATE TABLE t1(a PRIMARY KEY, b, c);
+ CREATE INDEX i1 ON t1(b);
+ CREATE INDEX i2 ON t1(c);
+
+ WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 )
+ INSERT INTO t1
+ SELECT randomblob(10), randomblob(100), randomblob(100) FROM s;
+ }
+}
+
+proc build_rbu {db} {
+ $db eval {
+ CREATE TABLE data_t1(a, b, c, rbu_control);
+ WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 )
+ INSERT INTO data_t1
+ SELECT randomblob(10), randomblob(100), randomblob(100), 0 FROM s;
+ }
+}
+
+proc step_rbu2 {bOpenClose openr1 openr2} {
+
+ forcedelete teststate.db1
+ forcedelete teststate.db2
+
+ if {$bOpenClose!=0 && $bOpenClose!=1} { error $bOpenClose }
+ if {$bOpenClose==0} {
+ eval $openr1
+ eval $openr2
+ }
+
+ set b1 0
+ set b2 0
+
+ while {$b1==0 || $b2==0} {
+ if {$bOpenClose==1} {
+ if {$b1==0} { eval $openr1 teststate.db1 }
+ if {$b2==0} { eval $openr2 teststate.db2 }
+ }
+ if {$b1==0} {
+ set rc1 [r1 step]
+ if {$rc1 != "SQLITE_OK"} { set b1 1 }
+ }
+ if {$b2==0} {
+ set rc2 [r2 step]
+ if {$rc2 != "SQLITE_OK"} { set b2 1 }
+ }
+ if {$bOpenClose==1} {
+ if {$b1==0} { r1 close }
+ if {$b2==0} { r2 close }
+ }
+ }
+
+ set rc1 [r1 close]
+ set rc2 [r2 close]
+
+ list $rc1 $rc2
+}
+
+
+for {set i 0} {$i<=3} {incr i} {
+
+ if {$i & 0x01} {
+ sqlite3rbu_create_vfs -default myrbu ""
+ }
+ set bOpenClose [expr $i>>1]
+
+ forcedelete test.db
+ forcedelete test.db2
+ forcedelete rbu.db
+ forcedelete rbu.db2
+
+ do_test 1.$i.0 {
+ sqlite3 db test.db
+ sqlite3 db2 test.db2
+ build_db db
+ build_db db2
+
+ sqlite3 rbu1 rbu.db
+ sqlite3 rbu2 rbu.db2
+
+ build_rbu rbu1
+ build_rbu rbu2
+
+ rbu1 close
+ rbu2 close
+ } {}
+
+ set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}]
+ set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}]
+
+ do_test 1.$i.1 {
+ step_rbu2 $bOpenClose {
+ sqlite3rbu r1 test.db rbu.db
+ } {
+ sqlite3rbu r2 test.db2 rbu.db2
+ }
+ } {SQLITE_DONE SQLITE_DONE}
+
+ do_execsql_test -db db 1.$i.2.1 { PRAGMA integrity_check } ok
+ do_execsql_test -db db2 1.$i.2.2 { PRAGMA integrity_check } ok
+
+ do_execsql_test -db db 1.$i.3.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 0
+ do_execsql_test -db db2 1.$i.3.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0
+
+ catch { db close }
+ catch { db2 close }
+ #-----------------------------------------------------------------------
+ forcedelete test.db2
+ forcedelete test.db
+ forcedelete rbu.db2
+
+ do_test 1.$i.4 {
+ sqlite3 db test.db
+ sqlite3 db2 test.db2
+ build_db db
+ build_db db2
+ sqlite3 rbu2 rbu.db2
+ build_rbu rbu2
+ rbu2 close
+ } {}
+
+ set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}]
+ set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}]
+
+ do_test 1.$i.5 {
+ step_rbu2 $bOpenClose {
+ sqlite3rbu_vacuum r1 test.db
+ } {
+ sqlite3rbu r2 test.db2 rbu.db2
+ }
+ } {SQLITE_DONE SQLITE_DONE}
+
+ do_execsql_test -db db 1.$i.6.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 1
+ do_execsql_test -db db2 1.$i.6.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0
+
+ do_execsql_test -db db 1.$i.7.1 { PRAGMA integrity_check } ok
+ do_execsql_test -db db2 1.$i.7.2 { PRAGMA integrity_check } ok
+
+ catch { db close }
+ catch { db2 close }
+ if {$i & 0x01} {
+ sqlite3rbu_destroy_vfs myrbu
+ }
+
+}
+
+
+finish_test
Index: ext/rbu/rbuprogress.test
==================================================================
--- ext/rbu/rbuprogress.test
+++ ext/rbu/rbuprogress.test
@@ -414,6 +414,5 @@
}
}
finish_test
-
Index: ext/rbu/rburesume.test
==================================================================
--- ext/rbu/rburesume.test
+++ ext/rbu/rburesume.test
@@ -249,6 +249,5 @@
} {60 ok}
db2 close
}
finish_test
-
Index: ext/rbu/rbusave.test
==================================================================
--- ext/rbu/rbusave.test
+++ ext/rbu/rbusave.test
@@ -100,6 +100,5 @@
SELECT * FROM t1;
SELECT * FROM t2;
} {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4}
finish_test
-
ADDED ext/rbu/rbusplit.test
Index: ext/rbu/rbusplit.test
==================================================================
--- /dev/null
+++ ext/rbu/rbusplit.test
@@ -0,0 +1,94 @@
+# 2018 April 28
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rbusplit
+
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+autoinstall_test_functions
+
+proc build_db {db} {
+ $db eval {
+ CREATE TABLE t1(a PRIMARY KEY, b, c);
+ CREATE TABLE t2(a PRIMARY KEY, b, c);
+
+ CREATE INDEX t1c ON t1(c);
+ }
+}
+
+proc build_rbu {filename} {
+ forcedelete $filename
+ sqlite3 dbRbu $filename
+ dbRbu eval {
+ CREATE TABLE data0_t1(a, b, c, rbu_control);
+ CREATE TABLE data1_t1(a, b, c, rbu_control);
+ CREATE TABLE data2_t1(a, b, c, rbu_control);
+ CREATE TABLE data3_t1(a, b, c, rbu_control);
+
+ CREATE TABLE data_t2(a, b, c, rbu_control);
+
+ INSERT INTO data0_t1 VALUES(1, 1, 1, 0);
+ INSERT INTO data0_t1 VALUES(2, 2, 2, 0);
+ INSERT INTO data0_t1 VALUES(3, 3, 3, 0);
+ INSERT INTO data0_t1 VALUES(4, 4, 4, 0);
+ INSERT INTO data1_t1 VALUES(5, 5, 5, 0);
+ INSERT INTO data1_t1 VALUES(6, 6, 6, 0);
+ INSERT INTO data1_t1 VALUES(7, 7, 7, 0);
+ INSERT INTO data1_t1 VALUES(8, 8, 8, 0);
+ INSERT INTO data3_t1 VALUES(9, 9, 9, 0);
+
+ INSERT INTO data_t2 VALUES(1, 1, 1, 0);
+ INSERT INTO data_t2 VALUES(2, 2, 2, 0);
+ INSERT INTO data_t2 VALUES(3, 3, 3, 0);
+ INSERT INTO data_t2 VALUES(4, 4, 4, 0);
+ INSERT INTO data_t2 VALUES(5, 5, 5, 0);
+ INSERT INTO data_t2 VALUES(6, 6, 6, 0);
+ INSERT INTO data_t2 VALUES(7, 7, 7, 0);
+ INSERT INTO data_t2 VALUES(8, 8, 8, 0);
+ INSERT INTO data_t2 VALUES(9, 9, 9, 0);
+ }
+
+ dbRbu close
+}
+
+foreach {tn cmd} {
+ 1 run_rbu
+ 2 step_rbu
+} {
+ reset_db
+ build_db db
+ build_rbu testrbu.db
+
+ do_test 1.$tn.1 {
+ $cmd test.db testrbu.db
+ } {SQLITE_DONE}
+ do_execsql_test 1.$tn.1 {
+ SELECT * FROM t1;
+ } {
+ 1 1 1 2 2 2 3 3 3 4 4 4
+ 5 5 5 6 6 6 7 7 7 8 8 8
+ 9 9 9
+ }
+ do_execsql_test 1.$tn.2 {
+ SELECT * FROM t2;
+ } {
+ 1 1 1 2 2 2 3 3 3 4 4 4
+ 5 5 5 6 6 6 7 7 7 8 8 8
+ 9 9 9
+ }
+}
+
+finish_test
Index: ext/rbu/rbutemplimit.test
==================================================================
--- ext/rbu/rbutemplimit.test
+++ ext/rbu/rbutemplimit.test
@@ -124,6 +124,5 @@
step_rbu_cachesize test.db test.db2 1000 10 1400000
} {1 SQLITE_FULL}
do_test 1.6.2 { info commands rbu } {}
finish_test
-
Index: ext/rbu/rbuvacuum.test
==================================================================
--- ext/rbu/rbuvacuum.test
+++ ext/rbu/rbuvacuum.test
@@ -395,6 +395,5 @@
list [catch { rbu close } msg] $msg
} {0 SQLITE_DONE}
catch { db close }
finish_test
-
Index: ext/rbu/rbuvacuum2.test
==================================================================
--- ext/rbu/rbuvacuum2.test
+++ ext/rbu/rbuvacuum2.test
@@ -230,6 +230,5 @@
rbu close
execsql { PRAGMA integrity_check }
} {ok}
finish_test
-
Index: ext/rbu/sqlite3rbu.c
==================================================================
--- ext/rbu/sqlite3rbu.c
+++ ext/rbu/sqlite3rbu.c
@@ -151,10 +151,14 @@
** Valid if STAGE==1. The current change-counter cookie value in the
** target db file.
**
** RBU_STATE_OALSZ:
** Valid if STAGE==1. The size in bytes of the *-oal file.
+**
+** RBU_STATE_DATATBL:
+** Only valid if STAGE==1. The RBU database name of the table
+** currently being read.
*/
#define RBU_STATE_STAGE 1
#define RBU_STATE_TBL 2
#define RBU_STATE_IDX 3
#define RBU_STATE_ROW 4
@@ -161,10 +165,11 @@
#define RBU_STATE_PROGRESS 5
#define RBU_STATE_CKPT 6
#define RBU_STATE_COOKIE 7
#define RBU_STATE_OALSZ 8
#define RBU_STATE_PHASEONESTEP 9
+#define RBU_STATE_DATATBL 10
#define RBU_STAGE_OAL 1
#define RBU_STAGE_MOVE 2
#define RBU_STAGE_CAPTURE 3
#define RBU_STAGE_CKPT 4
@@ -203,10 +208,11 @@
** A structure to store values read from the rbu_state table in memory.
*/
struct RbuState {
int eStage;
char *zTbl;
+ char *zDataTbl;
char *zIdx;
i64 iWalCksum;
int nRow;
i64 nProgress;
u32 iCookie;
@@ -397,11 +403,12 @@
struct rbu_vfs {
sqlite3_vfs base; /* rbu VFS shim methods */
sqlite3_vfs *pRealVfs; /* Underlying VFS */
sqlite3_mutex *mutex; /* Mutex to protect pMain */
sqlite3rbu *pRbu; /* Owner RBU object */
- rbu_file *pMain; /* Linked list of main db files */
+ rbu_file *pMain; /* List of main db files */
+ rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */
};
/*
** Each file opened by an rbu VFS is represented by an instance of
** the following structure.
@@ -426,10 +433,11 @@
char *zDel; /* Delete this when closing file */
const char *zWal; /* Wal filename for this main db file */
rbu_file *pWalFd; /* Wal file descriptor for this main db */
rbu_file *pMainNext; /* Next MAIN_DB file */
+ rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */
};
/*
** True for an RBU vacuum handle, or false otherwise.
*/
@@ -1804,11 +1812,11 @@
int bKey = sqlite3_column_int(pXInfo, 5);
if( bKey ){
int iCid = sqlite3_column_int(pXInfo, 1);
int bDesc = sqlite3_column_int(pXInfo, 3);
const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
- zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma,
+ zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma,
iCid, pIter->azTblType[iCid], zCollate
);
zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":"");
zComma = ", ";
}
@@ -1865,11 +1873,11 @@
if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){
/* If the target table column is an "INTEGER PRIMARY KEY", add
** "PRIMARY KEY" to the imposter table column declaration. */
zPk = "PRIMARY KEY ";
}
- zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s",
+ zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s",
zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl,
(pIter->abNotNull[iCol] ? " NOT NULL" : "")
);
zComma = ", ";
}
@@ -2266,10 +2274,11 @@
** Free an RbuState object allocated by rbuLoadState().
*/
static void rbuFreeState(RbuState *p){
if( p ){
sqlite3_free(p->zTbl);
+ sqlite3_free(p->zDataTbl);
sqlite3_free(p->zIdx);
sqlite3_free(p);
}
}
@@ -2335,10 +2344,14 @@
break;
case RBU_STATE_PHASEONESTEP:
pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
break;
+
+ case RBU_STATE_DATATBL:
+ pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc);
+ break;
default:
rc = SQLITE_CORRUPT;
break;
}
@@ -3110,21 +3123,23 @@
"(%d, %d), "
"(%d, %d), "
"(%d, %lld), "
"(%d, %lld), "
"(%d, %lld), "
- "(%d, %lld) ",
+ "(%d, %lld), "
+ "(%d, %Q) ",
p->zStateDb,
RBU_STATE_STAGE, eStage,
RBU_STATE_TBL, p->objiter.zTbl,
RBU_STATE_IDX, p->objiter.zIdx,
RBU_STATE_ROW, p->nStep,
RBU_STATE_PROGRESS, p->nProgress,
RBU_STATE_CKPT, p->iWalCksum,
RBU_STATE_COOKIE, (i64)pFd->iCookie,
RBU_STATE_OALSZ, p->iOalSz,
- RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
+ RBU_STATE_PHASEONESTEP, p->nPhaseOneStep,
+ RBU_STATE_DATATBL, p->objiter.zDataTbl
)
);
assert( pInsert==0 || rc==SQLITE_OK );
if( rc==SQLITE_OK ){
@@ -3376,11 +3391,12 @@
RbuObjIter *pIter = &p->objiter;
int rc = SQLITE_OK;
while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup
|| rbuStrCompare(pIter->zIdx, pState->zIdx)
- || rbuStrCompare(pIter->zTbl, pState->zTbl)
+ || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl))
+ || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl))
)){
rc = rbuObjIterNext(p, pIter);
}
if( rc==SQLITE_OK && !pIter->zTbl ){
@@ -4013,10 +4029,73 @@
pFd->sz = nNew;
assert( pRbu->szTemp>=0 );
if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL;
return SQLITE_OK;
}
+
+/*
+** Add an item to the main-db lists, if it is not already present.
+**
+** There are two main-db lists. One for all file descriptors, and one
+** for all file descriptors with rbu_file.pDb!=0. If the argument has
+** rbu_file.pDb!=0, then it is assumed to already be present on the
+** main list and is only added to the pDb!=0 list.
+*/
+static void rbuMainlistAdd(rbu_file *p){
+ rbu_vfs *pRbuVfs = p->pRbuVfs;
+ rbu_file *pIter;
+ assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) );
+ sqlite3_mutex_enter(pRbuVfs->mutex);
+ if( p->pRbu==0 ){
+ for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext);
+ p->pMainNext = pRbuVfs->pMain;
+ pRbuVfs->pMain = p;
+ }else{
+ for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){}
+ if( pIter==0 ){
+ p->pMainRbuNext = pRbuVfs->pMainRbu;
+ pRbuVfs->pMainRbu = p;
+ }
+ }
+ sqlite3_mutex_leave(pRbuVfs->mutex);
+}
+
+/*
+** Remove an item from the main-db lists.
+*/
+static void rbuMainlistRemove(rbu_file *p){
+ rbu_file **pp;
+ sqlite3_mutex_enter(p->pRbuVfs->mutex);
+ for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){}
+ if( *pp ) *pp = p->pMainNext;
+ p->pMainNext = 0;
+ for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){}
+ if( *pp ) *pp = p->pMainRbuNext;
+ p->pMainRbuNext = 0;
+ sqlite3_mutex_leave(p->pRbuVfs->mutex);
+}
+
+/*
+** Given that zWal points to a buffer containing a wal file name passed to
+** either the xOpen() or xAccess() VFS method, search the main-db list for
+** a file-handle opened by the same database connection on the corresponding
+** database file.
+**
+** If parameter bRbu is true, only search for file-descriptors with
+** rbu_file.pDb!=0.
+*/
+static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){
+ rbu_file *pDb;
+ sqlite3_mutex_enter(pRbuVfs->mutex);
+ if( bRbu ){
+ for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){}
+ }else{
+ for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
+ }
+ sqlite3_mutex_leave(pRbuVfs->mutex);
+ return pDb;
+}
/*
** Close an rbu file.
*/
static int rbuVfsClose(sqlite3_file *pFile){
@@ -4031,21 +4110,18 @@
sqlite3_free(p->apShm);
p->apShm = 0;
sqlite3_free(p->zDel);
if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
- rbu_file **pp;
- sqlite3_mutex_enter(p->pRbuVfs->mutex);
- for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext));
- *pp = p->pMainNext;
- sqlite3_mutex_leave(p->pRbuVfs->mutex);
+ rbuMainlistRemove(p);
rbuUnlockShm(p);
p->pReal->pMethods->xShmUnmap(p->pReal, 0);
}
else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){
rbuUpdateTempSize(p, 0);
}
+ assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p );
/* Close the underlying file handle */
rc = p->pReal->pMethods->xClose(p->pReal);
return rc;
}
@@ -4300,10 +4376,13 @@
rc = SQLITE_ERROR;
pRbu->zErrmsg = sqlite3_mprintf("rbu/zipvfs setup error");
}else if( rc==SQLITE_NOTFOUND ){
pRbu->pTargetFd = p;
p->pRbu = pRbu;
+ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){
+ rbuMainlistAdd(p);
+ }
if( p->pWalFd ) p->pWalFd->pRbu = pRbu;
rc = SQLITE_OK;
}
}
return rc;
@@ -4461,24 +4540,10 @@
rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag);
}
return rc;
}
-/*
-** Given that zWal points to a buffer containing a wal file name passed to
-** either the xOpen() or xAccess() VFS method, return a pointer to the
-** file-handle opened by the same database connection on the corresponding
-** database file.
-*/
-static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){
- rbu_file *pDb;
- sqlite3_mutex_enter(pRbuVfs->mutex);
- for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){}
- sqlite3_mutex_leave(pRbuVfs->mutex);
- return pDb;
-}
-
/*
** A main database named zName has just been opened. The following
** function returns a pointer to a buffer owned by SQLite that contains
** the name of the *-wal file this db connection will use. SQLite
** happens to pass a pointer to this buffer when using xAccess()
@@ -4553,11 +4618,11 @@
** happens to pass a pointer to this buffer when using xAccess()
** or xOpen() to operate on the *-wal file. */
pFd->zWal = rbuMainToWal(zName, flags);
}
else if( flags & SQLITE_OPEN_WAL ){
- rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName);
+ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0);
if( pDb ){
if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
/* This call is to open a *-wal file. Intead, open the *-oal. This
** code ensures that the string passed to xOpen() is terminated by a
** pair of '\0' bytes in case the VFS attempts to extract a URI
@@ -4605,14 +4670,11 @@
/* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods
** pointer and, if the file is a main database file, link it into the
** mutex protected linked list of all such files. */
pFile->pMethods = &rbuvfs_io_methods;
if( flags & SQLITE_OPEN_MAIN_DB ){
- sqlite3_mutex_enter(pRbuVfs->mutex);
- pFd->pMainNext = pRbuVfs->pMain;
- pRbuVfs->pMain = pFd;
- sqlite3_mutex_leave(pRbuVfs->mutex);
+ rbuMainlistAdd(pFd);
}
}else{
sqlite3_free(pFd->zDel);
}
@@ -4656,11 +4718,11 @@
** causing SQLite to call xOpen() to open it. This call will also
** be intercepted (see the rbuVfsOpen() function) and the *-oal
** file opened instead.
*/
if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){
- rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath);
+ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1);
if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){
if( *pResOut ){
rc = SQLITE_CANTOPEN;
}else{
sqlite3_int64 sz = 0;
Index: ext/rbu/test_rbu.c
==================================================================
--- ext/rbu/test_rbu.c
+++ ext/rbu/test_rbu.c
@@ -79,10 +79,11 @@
{"state", 2, ""}, /* 7 */
{"progress", 2, ""}, /* 8 */
{"close_no_error", 2, ""}, /* 9 */
{"temp_size_limit", 3, "LIMIT"}, /* 10 */
{"temp_size", 2, ""}, /* 11 */
+ {"dbRbu_eval", 3, "SQL"}, /* 12 */
{0,0,0}
};
int iCmd;
if( objc<2 ){
@@ -144,12 +145,13 @@
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1));
ret = (rc==SQLITE_OK ? TCL_OK : TCL_ERROR);
break;
}
- case 4: /* dbMain_eval */ {
- sqlite3 *db = sqlite3rbu_db(pRbu, 0);
+ case 12: /* dbRbu_eval */
+ case 4: /* dbMain_eval */ {
+ sqlite3 *db = sqlite3rbu_db(pRbu, (iCmd==12));
int rc = sqlite3_exec(db, Tcl_GetString(objv[2]), 0, 0, 0);
if( rc!=SQLITE_OK ){
Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3_errmsg(db), -1));
ret = TCL_ERROR;
}
ADDED ext/repair/README.md
Index: ext/repair/README.md
==================================================================
--- /dev/null
+++ ext/repair/README.md
@@ -0,0 +1,16 @@
+This folder contains extensions and utility programs intended to analyze
+live database files, detect problems, and possibly fix them.
+
+As SQLite is being used on larger and larger databases, database sizes
+are growing into the terabyte range. At that size, hardware malfunctions
+and/or cosmic rays will occasionally corrupt a database file. Detecting
+problems and fixing errors a terabyte-sized databases can take hours or days,
+and it is undesirable to take applications that depend on the databases
+off-line for such a long time.
+The utilities in the folder are intended to provide mechanisms for
+detecting and fixing problems in large databases while those databases
+are in active use.
+
+The utilities and extensions in this folder are experimental and under
+active development at the time of this writing (2017-10-12). If and when
+they stabilize, this README will be updated to reflect that fact.
ADDED ext/repair/checkfreelist.c
Index: ext/repair/checkfreelist.c
==================================================================
--- /dev/null
+++ ext/repair/checkfreelist.c
@@ -0,0 +1,299 @@
+/*
+** 2017 October 11
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This module exports a single C function:
+**
+** int sqlite3_check_freelist(sqlite3 *db, const char *zDb);
+**
+** This function checks the free-list in database zDb (one of "main",
+** "temp", etc.) and reports any errors by invoking the sqlite3_log()
+** function. It returns SQLITE_OK if successful, or an SQLite error
+** code otherwise. It is not an error if the free-list is corrupted but
+** no IO or OOM errors occur.
+**
+** If this file is compiled and loaded as an SQLite loadable extension,
+** it adds an SQL function "checkfreelist" to the database handle, to
+** be invoked as follows:
+**
+** SELECT checkfreelist();
+**
+** This function performs the same checks as sqlite3_check_freelist(),
+** except that it returns all error messages as a single text value,
+** separated by newline characters. If the freelist is not corrupted
+** in any way, an empty string is returned.
+**
+** To compile this module for use as an SQLite loadable extension:
+**
+** gcc -Os -fPIC -shared checkfreelist.c -o checkfreelist.so
+*/
+
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+#ifndef SQLITE_AMALGAMATION
+# include
+# include
+# include
+# include
+# define ALWAYS(X) 1
+# define NEVER(X) 0
+ typedef unsigned char u8;
+ typedef unsigned short u16;
+ typedef unsigned int u32;
+#define get4byte(x) ( \
+ ((u32)((x)[0])<<24) + \
+ ((u32)((x)[1])<<16) + \
+ ((u32)((x)[2])<<8) + \
+ ((u32)((x)[3])) \
+)
+#endif
+
+/*
+** Execute a single PRAGMA statement and return the integer value returned
+** via output parameter (*pnOut).
+**
+** The SQL statement passed as the third argument should be a printf-style
+** format string containing a single "%s" which will be replace by the
+** value passed as the second argument. e.g.
+**
+** sqlGetInteger(db, "main", "PRAGMA %s.page_count", pnOut)
+**
+** executes "PRAGMA main.page_count" and stores the results in (*pnOut).
+*/
+static int sqlGetInteger(
+ sqlite3 *db, /* Database handle */
+ const char *zDb, /* Database name ("main", "temp" etc.) */
+ const char *zFmt, /* SQL statement format */
+ u32 *pnOut /* OUT: Integer value */
+){
+ int rc, rc2;
+ char *zSql;
+ sqlite3_stmt *pStmt = 0;
+ int bOk = 0;
+
+ zSql = sqlite3_mprintf(zFmt, zDb);
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+ sqlite3_free(zSql);
+ }
+
+ if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
+ *pnOut = (u32)sqlite3_column_int(pStmt, 0);
+ bOk = 1;
+ }
+
+ rc2 = sqlite3_finalize(pStmt);
+ if( rc==SQLITE_OK ) rc = rc2;
+ if( rc==SQLITE_OK && bOk==0 ) rc = SQLITE_ERROR;
+ return rc;
+}
+
+/*
+** Argument zFmt must be a printf-style format string and must be
+** followed by its required arguments. If argument pzOut is NULL,
+** then the results of printf()ing the format string are passed to
+** sqlite3_log(). Otherwise, they are appended to the string
+** at (*pzOut).
+*/
+static int checkFreelistError(char **pzOut, const char *zFmt, ...){
+ int rc = SQLITE_OK;
+ char *zErr = 0;
+ va_list ap;
+
+ va_start(ap, zFmt);
+ zErr = sqlite3_vmprintf(zFmt, ap);
+ if( zErr==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ if( pzOut ){
+ *pzOut = sqlite3_mprintf("%s%z%s", *pzOut?"\n":"", *pzOut, zErr);
+ if( *pzOut==0 ) rc = SQLITE_NOMEM;
+ }else{
+ sqlite3_log(SQLITE_ERROR, "checkfreelist: %s", zErr);
+ }
+ sqlite3_free(zErr);
+ }
+ va_end(ap);
+ return rc;
+}
+
+static int checkFreelist(
+ sqlite3 *db,
+ const char *zDb,
+ char **pzOut
+){
+ /* This query returns one row for each page on the free list. Each row has
+ ** two columns - the page number and page content. */
+ const char *zTrunk =
+ "WITH freelist_trunk(i, d, n) AS ("
+ "SELECT 1, NULL, sqlite_readint32(data, 32) "
+ "FROM sqlite_dbpage(:1) WHERE pgno=1 "
+ "UNION ALL "
+ "SELECT n, data, sqlite_readint32(data) "
+ "FROM freelist_trunk, sqlite_dbpage(:1) WHERE pgno=n "
+ ")"
+ "SELECT i, d FROM freelist_trunk WHERE i!=1;";
+
+ int rc, rc2; /* Return code */
+ sqlite3_stmt *pTrunk = 0; /* Compilation of zTrunk */
+ u32 nPage = 0; /* Number of pages in db */
+ u32 nExpected = 0; /* Expected number of free pages */
+ u32 nFree = 0; /* Number of pages on free list */
+
+ if( zDb==0 ) zDb = "main";
+
+ if( (rc = sqlGetInteger(db, zDb, "PRAGMA %s.page_count", &nPage))
+ || (rc = sqlGetInteger(db, zDb, "PRAGMA %s.freelist_count", &nExpected))
+ ){
+ return rc;
+ }
+
+ rc = sqlite3_prepare_v2(db, zTrunk, -1, &pTrunk, 0);
+ if( rc!=SQLITE_OK ) return rc;
+ sqlite3_bind_text(pTrunk, 1, zDb, -1, SQLITE_STATIC);
+ while( rc==SQLITE_OK && sqlite3_step(pTrunk)==SQLITE_ROW ){
+ u32 i;
+ u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0);
+ const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1);
+ u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1);
+ u32 iNext = get4byte(&aData[0]);
+ u32 nLeaf = get4byte(&aData[4]);
+
+ if( nLeaf>((nData/4)-2-6) ){
+ rc = checkFreelistError(pzOut,
+ "leaf count out of range (%d) on trunk page %d",
+ (int)nLeaf, (int)iTrunk
+ );
+ nLeaf = (nData/4) - 2 - 6;
+ }
+
+ nFree += 1+nLeaf;
+ if( iNext>nPage ){
+ rc = checkFreelistError(pzOut,
+ "trunk page %d is out of range", (int)iNext
+ );
+ }
+
+ for(i=0; rc==SQLITE_OK && inPage ){
+ rc = checkFreelistError(pzOut,
+ "leaf page %d is out of range (child %d of trunk page %d)",
+ (int)iLeaf, (int)i, (int)iTrunk
+ );
+ }
+ }
+ }
+
+ if( rc==SQLITE_OK && nFree!=nExpected ){
+ rc = checkFreelistError(pzOut,
+ "free-list count mismatch: actual=%d header=%d",
+ (int)nFree, (int)nExpected
+ );
+ }
+
+ rc2 = sqlite3_finalize(pTrunk);
+ if( rc==SQLITE_OK ) rc = rc2;
+ return rc;
+}
+
+int sqlite3_check_freelist(sqlite3 *db, const char *zDb){
+ return checkFreelist(db, zDb, 0);
+}
+
+static void checkfreelist_function(
+ sqlite3_context *pCtx,
+ int nArg,
+ sqlite3_value **apArg
+){
+ const char *zDb;
+ int rc;
+ char *zOut = 0;
+ sqlite3 *db = sqlite3_context_db_handle(pCtx);
+
+ assert( nArg==1 );
+ zDb = (const char*)sqlite3_value_text(apArg[0]);
+ rc = checkFreelist(db, zDb, &zOut);
+ if( rc==SQLITE_OK ){
+ sqlite3_result_text(pCtx, zOut?zOut:"ok", -1, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_result_error_code(pCtx, rc);
+ }
+
+ sqlite3_free(zOut);
+}
+
+/*
+** An SQL function invoked as follows:
+**
+** sqlite_readint32(BLOB) -- Decode 32-bit integer from start of blob
+*/
+static void readint_function(
+ sqlite3_context *pCtx,
+ int nArg,
+ sqlite3_value **apArg
+){
+ const u8 *zBlob;
+ int nBlob;
+ int iOff = 0;
+ u32 iRet = 0;
+
+ if( nArg!=1 && nArg!=2 ){
+ sqlite3_result_error(
+ pCtx, "wrong number of arguments to function sqlite_readint32()", -1
+ );
+ return;
+ }
+ if( nArg==2 ){
+ iOff = sqlite3_value_int(apArg[1]);
+ }
+
+ zBlob = sqlite3_value_blob(apArg[0]);
+ nBlob = sqlite3_value_bytes(apArg[0]);
+
+ if( nBlob>=(iOff+4) ){
+ iRet = get4byte(&zBlob[iOff]);
+ }
+
+ sqlite3_result_int64(pCtx, (sqlite3_int64)iRet);
+}
+
+/*
+** Register the SQL functions.
+*/
+static int cflRegister(sqlite3 *db){
+ int rc = sqlite3_create_function(
+ db, "sqlite_readint32", -1, SQLITE_UTF8, 0, readint_function, 0, 0
+ );
+ if( rc!=SQLITE_OK ) return rc;
+ rc = sqlite3_create_function(
+ db, "checkfreelist", 1, SQLITE_UTF8, 0, checkfreelist_function, 0, 0
+ );
+ return rc;
+}
+
+/*
+** Extension load function.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_checkfreelist_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi);
+ return cflRegister(db);
+}
ADDED ext/repair/checkindex.c
Index: ext/repair/checkindex.c
==================================================================
--- /dev/null
+++ ext/repair/checkindex.c
@@ -0,0 +1,927 @@
+/*
+** 2017 October 27
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT1
+
+/*
+** Stuff that is available inside the amalgamation, but which we need to
+** declare ourselves if this module is compiled separately.
+*/
+#ifndef SQLITE_AMALGAMATION
+# include
+# include
+# include
+# include
+typedef unsigned char u8;
+typedef unsigned short u16;
+typedef unsigned int u32;
+#define get4byte(x) ( \
+ ((u32)((x)[0])<<24) + \
+ ((u32)((x)[1])<<16) + \
+ ((u32)((x)[2])<<8) + \
+ ((u32)((x)[3])) \
+)
+#endif
+
+typedef struct CidxTable CidxTable;
+typedef struct CidxCursor CidxCursor;
+
+struct CidxTable {
+ sqlite3_vtab base; /* Base class. Must be first */
+ sqlite3 *db;
+};
+
+struct CidxCursor {
+ sqlite3_vtab_cursor base; /* Base class. Must be first */
+ sqlite3_int64 iRowid; /* Row number of the output */
+ char *zIdxName; /* Copy of the index_name parameter */
+ char *zAfterKey; /* Copy of the after_key parameter */
+ sqlite3_stmt *pStmt; /* SQL statement that generates the output */
+};
+
+typedef struct CidxColumn CidxColumn;
+struct CidxColumn {
+ char *zExpr; /* Text for indexed expression */
+ int bDesc; /* True for DESC columns, otherwise false */
+ int bKey; /* Part of index, not PK */
+};
+
+typedef struct CidxIndex CidxIndex;
+struct CidxIndex {
+ char *zWhere; /* WHERE clause, if any */
+ int nCol; /* Elements in aCol[] array */
+ CidxColumn aCol[1]; /* Array of indexed columns */
+};
+
+static void *cidxMalloc(int *pRc, int n){
+ void *pRet = 0;
+ assert( n!=0 );
+ if( *pRc==SQLITE_OK ){
+ pRet = sqlite3_malloc(n);
+ if( pRet ){
+ memset(pRet, 0, n);
+ }else{
+ *pRc = SQLITE_NOMEM;
+ }
+ }
+ return pRet;
+}
+
+static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){
+ va_list ap;
+ va_start(ap, zFmt);
+ assert( pCsr->base.pVtab->zErrMsg==0 );
+ pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap);
+ va_end(ap);
+}
+
+/*
+** Connect to the incremental_index_check virtual table.
+*/
+static int cidxConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ int rc = SQLITE_OK;
+ CidxTable *pRet;
+
+#define IIC_ERRMSG 0
+#define IIC_CURRENT_KEY 1
+#define IIC_INDEX_NAME 2
+#define IIC_AFTER_KEY 3
+#define IIC_SCANNER_SQL 4
+ rc = sqlite3_declare_vtab(db,
+ "CREATE TABLE xyz("
+ " errmsg TEXT," /* Error message or NULL if everything is ok */
+ " current_key TEXT," /* SQLite quote() text of key values */
+ " index_name HIDDEN," /* IN: name of the index being scanned */
+ " after_key HIDDEN," /* IN: Start scanning after this key */
+ " scanner_sql HIDDEN" /* debuggingn info: SQL used for scanner */
+ ")"
+ );
+ pRet = cidxMalloc(&rc, sizeof(CidxTable));
+ if( pRet ){
+ pRet->db = db;
+ }
+
+ *ppVtab = (sqlite3_vtab*)pRet;
+ return rc;
+}
+
+/*
+** Disconnect from or destroy an incremental_index_check virtual table.
+*/
+static int cidxDisconnect(sqlite3_vtab *pVtab){
+ CidxTable *pTab = (CidxTable*)pVtab;
+ sqlite3_free(pTab);
+ return SQLITE_OK;
+}
+
+/*
+** idxNum and idxStr are not used. There are only three possible plans,
+** which are all distinguished by the number of parameters.
+**
+** No parameters: A degenerate plan. The result is zero rows.
+** 1 Parameter: Scan all of the index starting with first entry
+** 2 parameters: Scan the index starting after the "after_key".
+**
+** Provide successively smaller costs for each of these plans to encourage
+** the query planner to select the one with the most parameters.
+*/
+static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){
+ int iIdxName = -1;
+ int iAfterKey = -1;
+ int i;
+
+ for(i=0; inConstraint; i++){
+ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
+ if( p->usable==0 ) continue;
+ if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue;
+
+ if( p->iColumn==IIC_INDEX_NAME ){
+ iIdxName = i;
+ }
+ if( p->iColumn==IIC_AFTER_KEY ){
+ iAfterKey = i;
+ }
+ }
+
+ if( iIdxName<0 ){
+ pInfo->estimatedCost = 1000000000.0;
+ }else{
+ pInfo->aConstraintUsage[iIdxName].argvIndex = 1;
+ pInfo->aConstraintUsage[iIdxName].omit = 1;
+ if( iAfterKey<0 ){
+ pInfo->estimatedCost = 1000000.0;
+ }else{
+ pInfo->aConstraintUsage[iAfterKey].argvIndex = 2;
+ pInfo->aConstraintUsage[iAfterKey].omit = 1;
+ pInfo->estimatedCost = 1000.0;
+ }
+ }
+
+ return SQLITE_OK;
+}
+
+/*
+** Open a new btreeinfo cursor.
+*/
+static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+ CidxCursor *pRet;
+ int rc = SQLITE_OK;
+
+ pRet = cidxMalloc(&rc, sizeof(CidxCursor));
+
+ *ppCursor = (sqlite3_vtab_cursor*)pRet;
+ return rc;
+}
+
+/*
+** Close a btreeinfo cursor.
+*/
+static int cidxClose(sqlite3_vtab_cursor *pCursor){
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ sqlite3_finalize(pCsr->pStmt);
+ sqlite3_free(pCsr->zIdxName);
+ sqlite3_free(pCsr->zAfterKey);
+ sqlite3_free(pCsr);
+ return SQLITE_OK;
+}
+
+/*
+** Move a btreeinfo cursor to the next entry in the file.
+*/
+static int cidxNext(sqlite3_vtab_cursor *pCursor){
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ int rc = sqlite3_step(pCsr->pStmt);
+ if( rc!=SQLITE_ROW ){
+ rc = sqlite3_finalize(pCsr->pStmt);
+ pCsr->pStmt = 0;
+ if( rc!=SQLITE_OK ){
+ sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db;
+ cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db));
+ }
+ }else{
+ pCsr->iRowid++;
+ rc = SQLITE_OK;
+ }
+ return rc;
+}
+
+/* We have reached EOF if previous sqlite3_step() returned
+** anything other than SQLITE_ROW;
+*/
+static int cidxEof(sqlite3_vtab_cursor *pCursor){
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ return pCsr->pStmt==0;
+}
+
+static char *cidxMprintf(int *pRc, const char *zFmt, ...){
+ char *zRet = 0;
+ va_list ap;
+ va_start(ap, zFmt);
+ zRet = sqlite3_vmprintf(zFmt, ap);
+ if( *pRc==SQLITE_OK ){
+ if( zRet==0 ){
+ *pRc = SQLITE_NOMEM;
+ }
+ }else{
+ sqlite3_free(zRet);
+ zRet = 0;
+ }
+ va_end(ap);
+ return zRet;
+}
+
+static sqlite3_stmt *cidxPrepare(
+ int *pRc, CidxCursor *pCsr, const char *zFmt, ...
+){
+ sqlite3_stmt *pRet = 0;
+ char *zSql;
+ va_list ap; /* ... printf arguments */
+ va_start(ap, zFmt);
+
+ zSql = sqlite3_vmprintf(zFmt, ap);
+ if( *pRc==SQLITE_OK ){
+ if( zSql==0 ){
+ *pRc = SQLITE_NOMEM;
+ }else{
+ sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db;
+ *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0);
+ if( *pRc!=SQLITE_OK ){
+ cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db));
+ }
+ }
+ }
+ sqlite3_free(zSql);
+ va_end(ap);
+
+ return pRet;
+}
+
+static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){
+ int rc = sqlite3_finalize(pStmt);
+ if( *pRc==SQLITE_OK ) *pRc = rc;
+}
+
+char *cidxStrdup(int *pRc, const char *zStr){
+ char *zRet = 0;
+ if( *pRc==SQLITE_OK ){
+ int n = (int)strlen(zStr);
+ zRet = cidxMalloc(pRc, n+1);
+ if( zRet ) memcpy(zRet, zStr, n+1);
+ }
+ return zRet;
+}
+
+static void cidxFreeIndex(CidxIndex *pIdx){
+ if( pIdx ){
+ int i;
+ for(i=0; inCol; i++){
+ sqlite3_free(pIdx->aCol[i].zExpr);
+ }
+ sqlite3_free(pIdx->zWhere);
+ sqlite3_free(pIdx);
+ }
+}
+
+static int cidx_isspace(char c){
+ return c==' ' || c=='\t' || c=='\r' || c=='\n';
+}
+
+static int cidx_isident(char c){
+ return c<0
+ || (c>='0' && c<='9') || (c>='a' && c<='z')
+ || (c>='A' && c<='Z') || c=='_';
+}
+
+#define CIDX_PARSE_EOF 0
+#define CIDX_PARSE_COMMA 1 /* "," */
+#define CIDX_PARSE_OPEN 2 /* "(" */
+#define CIDX_PARSE_CLOSE 3 /* ")" */
+
+/*
+** Argument zIn points into the start, middle or end of a CREATE INDEX
+** statement. If argument pbDoNotTrim is non-NULL, then this function
+** scans the input until it finds EOF, a comma (",") or an open or
+** close parenthesis character. It then sets (*pzOut) to point to said
+** character and returns a CIDX_PARSE_XXX constant as appropriate. The
+** parser is smart enough that special characters inside SQL strings
+** or comments are not returned for.
+**
+** Or, if argument pbDoNotTrim is NULL, then this function sets *pzOut
+** to point to the first character of the string that is not whitespace
+** or part of an SQL comment and returns CIDX_PARSE_EOF.
+**
+** Additionally, if pbDoNotTrim is not NULL and the element immediately
+** before (*pzOut) is an SQL comment of the form "-- comment", then
+** (*pbDoNotTrim) is set before returning. In all other cases it is
+** cleared.
+*/
+static int cidxFindNext(
+ const char *zIn,
+ const char **pzOut,
+ int *pbDoNotTrim /* OUT: True if prev is -- comment */
+){
+ const char *z = zIn;
+
+ while( 1 ){
+ while( cidx_isspace(*z) ) z++;
+ if( z[0]=='-' && z[1]=='-' ){
+ z += 2;
+ while( z[0]!='\n' ){
+ if( z[0]=='\0' ) return CIDX_PARSE_EOF;
+ z++;
+ }
+ while( cidx_isspace(*z) ) z++;
+ if( pbDoNotTrim ) *pbDoNotTrim = 1;
+ }else
+ if( z[0]=='/' && z[1]=='*' ){
+ z += 2;
+ while( z[0]!='*' || z[1]!='/' ){
+ if( z[1]=='\0' ) return CIDX_PARSE_EOF;
+ z++;
+ }
+ z += 2;
+ }else{
+ *pzOut = z;
+ if( pbDoNotTrim==0 ) return CIDX_PARSE_EOF;
+ switch( *z ){
+ case '\0':
+ return CIDX_PARSE_EOF;
+ case '(':
+ return CIDX_PARSE_OPEN;
+ case ')':
+ return CIDX_PARSE_CLOSE;
+ case ',':
+ return CIDX_PARSE_COMMA;
+
+ case '"':
+ case '\'':
+ case '`': {
+ char q = *z;
+ z++;
+ while( *z ){
+ if( *z==q ){
+ z++;
+ if( *z!=q ) break;
+ }
+ z++;
+ }
+ break;
+ }
+
+ case '[':
+ while( *z++!=']' );
+ break;
+
+ default:
+ z++;
+ break;
+ }
+ *pbDoNotTrim = 0;
+ }
+ }
+
+ assert( 0 );
+ return -1;
+}
+
+static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){
+ const char *z = zSql;
+ const char *z1;
+ int e;
+ int rc = SQLITE_OK;
+ int nParen = 1;
+ int bDoNotTrim = 0;
+ CidxColumn *pCol = pIdx->aCol;
+
+ e = cidxFindNext(z, &z, &bDoNotTrim);
+ if( e!=CIDX_PARSE_OPEN ) goto parse_error;
+ z1 = z+1;
+ z++;
+ while( nParen>0 ){
+ e = cidxFindNext(z, &z, &bDoNotTrim);
+ if( e==CIDX_PARSE_EOF ) goto parse_error;
+ if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){
+ const char *z2 = z;
+ if( pCol->zExpr ) goto parse_error;
+
+ if( bDoNotTrim==0 ){
+ while( cidx_isspace(z[-1]) ) z--;
+ if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){
+ z -= 3;
+ while( cidx_isspace(z[-1]) ) z--;
+ }else
+ if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){
+ z -= 4;
+ while( cidx_isspace(z[-1]) ) z--;
+ }
+ while( cidx_isspace(z1[0]) ) z1++;
+ }
+
+ pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1);
+ pCol++;
+ z = z1 = z2+1;
+ }
+ if( e==CIDX_PARSE_OPEN ) nParen++;
+ if( e==CIDX_PARSE_CLOSE ) nParen--;
+ z++;
+ }
+
+ /* Search for a WHERE clause */
+ cidxFindNext(z, &z, 0);
+ if( 0==sqlite3_strnicmp(z, "where", 5) ){
+ pIdx->zWhere = cidxMprintf(&rc, "%s\n", &z[5]);
+ }else if( z[0]!='\0' ){
+ goto parse_error;
+ }
+
+ return rc;
+
+ parse_error:
+ cidxCursorError(pCsr, "Parse error in: %s", zSql);
+ return SQLITE_ERROR;
+}
+
+static int cidxLookupIndex(
+ CidxCursor *pCsr, /* Cursor object */
+ const char *zIdx, /* Name of index to look up */
+ CidxIndex **ppIdx, /* OUT: Description of columns */
+ char **pzTab /* OUT: Table name */
+){
+ int rc = SQLITE_OK;
+ char *zTab = 0;
+ CidxIndex *pIdx = 0;
+
+ sqlite3_stmt *pFindTab = 0;
+ sqlite3_stmt *pInfo = 0;
+
+ /* Find the table for this index. */
+ pFindTab = cidxPrepare(&rc, pCsr,
+ "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'",
+ zIdx
+ );
+ if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){
+ const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1);
+ zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0));
+
+ pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx);
+ if( rc==SQLITE_OK ){
+ int nAlloc = 0;
+ int iCol = 0;
+
+ while( sqlite3_step(pInfo)==SQLITE_ROW ){
+ const char *zName = (const char*)sqlite3_column_text(pInfo, 2);
+ const char *zColl = (const char*)sqlite3_column_text(pInfo, 4);
+ CidxColumn *p;
+ if( zName==0 ) zName = "rowid";
+ if( iCol==nAlloc ){
+ int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8);
+ pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte);
+ nAlloc += 8;
+ }
+ p = &pIdx->aCol[iCol++];
+ p->bDesc = sqlite3_column_int(pInfo, 3);
+ p->bKey = sqlite3_column_int(pInfo, 5);
+ if( zSql==0 || p->bKey==0 ){
+ p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl);
+ }else{
+ p->zExpr = 0;
+ }
+ pIdx->nCol = iCol;
+ pIdx->zWhere = 0;
+ }
+ cidxFinalize(&rc, pInfo);
+ }
+
+ if( rc==SQLITE_OK && zSql ){
+ rc = cidxParseSQL(pCsr, pIdx, zSql);
+ }
+ }
+
+ cidxFinalize(&rc, pFindTab);
+ if( rc==SQLITE_OK && zTab==0 ){
+ rc = SQLITE_ERROR;
+ }
+
+ if( rc!=SQLITE_OK ){
+ sqlite3_free(zTab);
+ cidxFreeIndex(pIdx);
+ }else{
+ *pzTab = zTab;
+ *ppIdx = pIdx;
+ }
+
+ return rc;
+}
+
+static int cidxDecodeAfter(
+ CidxCursor *pCsr,
+ int nCol,
+ const char *zAfterKey,
+ char ***pazAfter
+){
+ char **azAfter;
+ int rc = SQLITE_OK;
+ int nAfterKey = (int)strlen(zAfterKey);
+
+ azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1);
+ if( rc==SQLITE_OK ){
+ int i;
+ char *zCopy = (char*)&azAfter[nCol];
+ char *p = zCopy;
+ memcpy(zCopy, zAfterKey, nAfterKey+1);
+ for(i=0; i='0' && *p<='9')
+ || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E'
+ ){
+ p++;
+ }
+ }
+
+ while( *p==' ' ) p++;
+ if( *p!=(i==(nCol-1) ? '\0' : ',') ){
+ goto parse_error;
+ }
+ *p++ = '\0';
+ }
+ }
+
+ *pazAfter = azAfter;
+ return rc;
+
+ parse_error:
+ sqlite3_free(azAfter);
+ *pazAfter = 0;
+ cidxCursorError(pCsr, "%s", "error parsing after value");
+ return SQLITE_ERROR;
+}
+
+static char *cidxWhere(
+ int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull
+){
+ char *zRet = 0;
+ const char *zSep = "";
+ int i;
+
+ for(i=0; i"),
+ azAfter[iGt]
+ );
+ }else{
+ zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr);
+ }
+
+ return zRet;
+}
+
+#define CIDX_CLIST_ALL 0
+#define CIDX_CLIST_ORDERBY 1
+#define CIDX_CLIST_CURRENT_KEY 2
+#define CIDX_CLIST_SUBWHERE 3
+#define CIDX_CLIST_SUBEXPR 4
+
+/*
+** This function returns various strings based on the contents of the
+** CidxIndex structure and the eType parameter.
+*/
+static char *cidxColumnList(
+ int *pRc, /* IN/OUT: Error code */
+ const char *zIdx,
+ CidxIndex *pIdx, /* Indexed columns */
+ int eType /* True to include ASC/DESC */
+){
+ char *zRet = 0;
+ if( *pRc==SQLITE_OK ){
+ const char *aDir[2] = {"", " DESC"};
+ int i;
+ const char *zSep = "";
+
+ for(i=0; inCol; i++){
+ CidxColumn *p = &pIdx->aCol[i];
+ assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 );
+ switch( eType ){
+
+ case CIDX_CLIST_ORDERBY:
+ zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]);
+ zSep = ",";
+ break;
+
+ case CIDX_CLIST_CURRENT_KEY:
+ zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i);
+ zSep = "||','||";
+ break;
+
+ case CIDX_CLIST_SUBWHERE:
+ if( p->bKey==0 ){
+ zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet,
+ zSep, p->zExpr, i
+ );
+ zSep = " AND ";
+ }
+ break;
+
+ case CIDX_CLIST_SUBEXPR:
+ if( p->bKey==1 ){
+ zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet,
+ zSep, p->zExpr, i
+ );
+ zSep = " AND ";
+ }
+ break;
+
+ default:
+ assert( eType==CIDX_CLIST_ALL );
+ zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i);
+ zSep = ", ";
+ break;
+ }
+ }
+ }
+
+ return zRet;
+}
+
+/*
+** Generate SQL (in memory obtained from sqlite3_malloc()) that will
+** continue the index scan for zIdxName starting after zAfterKey.
+*/
+int cidxGenerateScanSql(
+ CidxCursor *pCsr, /* The cursor which needs the new statement */
+ const char *zIdxName, /* index to be scanned */
+ const char *zAfterKey, /* start after this key, if not NULL */
+ char **pzSqlOut /* OUT: Write the generated SQL here */
+){
+ int rc;
+ char *zTab = 0;
+ char *zCurrentKey = 0;
+ char *zOrderBy = 0;
+ char *zSubWhere = 0;
+ char *zSubExpr = 0;
+ char *zSrcList = 0;
+ char **azAfter = 0;
+ CidxIndex *pIdx = 0;
+
+ *pzSqlOut = 0;
+ rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab);
+
+ zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY);
+ zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY);
+ zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE);
+ zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR);
+ zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL);
+
+ if( rc==SQLITE_OK && zAfterKey ){
+ rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter);
+ }
+
+ if( rc==SQLITE_OK ){
+ if( zAfterKey==0 ){
+ *pzSqlOut = cidxMprintf(&rc,
+ "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s "
+ "FROM (SELECT %s FROM %Q INDEXED BY %Q %s%sORDER BY %s) AS i",
+ zSubExpr, zTab, zSubWhere, zCurrentKey,
+ zSrcList, zTab, zIdxName,
+ (pIdx->zWhere ? "WHERE " : ""), (pIdx->zWhere ? pIdx->zWhere : ""),
+ zOrderBy
+ );
+ }else{
+ const char *zSep = "";
+ char *zSql;
+ int i;
+
+ zSql = cidxMprintf(&rc,
+ "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (",
+ zSubExpr, zTab, zSubWhere, zCurrentKey
+ );
+ for(i=pIdx->nCol-1; i>=0; i--){
+ int j;
+ if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue;
+ for(j=0; j<2; j++){
+ char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j);
+ zSql = cidxMprintf(&rc, "%z"
+ "%sSELECT * FROM ("
+ "SELECT %s FROM %Q INDEXED BY %Q WHERE %s%s%z ORDER BY %s"
+ ")",
+ zSql, zSep, zSrcList, zTab, zIdxName,
+ pIdx->zWhere ? pIdx->zWhere : "",
+ pIdx->zWhere ? " AND " : "",
+ zWhere, zOrderBy
+ );
+ zSep = " UNION ALL ";
+ if( pIdx->aCol[i].bDesc==0 ) break;
+ }
+ }
+ *pzSqlOut = cidxMprintf(&rc, "%z) AS i", zSql);
+ }
+ }
+
+ sqlite3_free(zTab);
+ sqlite3_free(zCurrentKey);
+ sqlite3_free(zOrderBy);
+ sqlite3_free(zSubWhere);
+ sqlite3_free(zSubExpr);
+ sqlite3_free(zSrcList);
+ cidxFreeIndex(pIdx);
+ sqlite3_free(azAfter);
+ return rc;
+}
+
+
+/*
+** Position a cursor back to the beginning.
+*/
+static int cidxFilter(
+ sqlite3_vtab_cursor *pCursor,
+ int idxNum, const char *idxStr,
+ int argc, sqlite3_value **argv
+){
+ int rc = SQLITE_OK;
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ const char *zIdxName = 0;
+ const char *zAfterKey = 0;
+
+ sqlite3_free(pCsr->zIdxName);
+ pCsr->zIdxName = 0;
+ sqlite3_free(pCsr->zAfterKey);
+ pCsr->zAfterKey = 0;
+ sqlite3_finalize(pCsr->pStmt);
+ pCsr->pStmt = 0;
+
+ if( argc>0 ){
+ zIdxName = (const char*)sqlite3_value_text(argv[0]);
+ if( argc>1 ){
+ zAfterKey = (const char*)sqlite3_value_text(argv[1]);
+ }
+ }
+
+ if( zIdxName ){
+ char *zSql = 0;
+ pCsr->zIdxName = sqlite3_mprintf("%s", zIdxName);
+ pCsr->zAfterKey = zAfterKey ? sqlite3_mprintf("%s", zAfterKey) : 0;
+ rc = cidxGenerateScanSql(pCsr, zIdxName, zAfterKey, &zSql);
+ if( zSql ){
+ pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql);
+ }
+ }
+
+ if( pCsr->pStmt ){
+ assert( rc==SQLITE_OK );
+ rc = cidxNext(pCursor);
+ }
+ pCsr->iRowid = 1;
+ return rc;
+}
+
+/*
+** Return a column value.
+*/
+static int cidxColumn(
+ sqlite3_vtab_cursor *pCursor,
+ sqlite3_context *ctx,
+ int iCol
+){
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ assert( iCol>=IIC_ERRMSG && iCol<=IIC_SCANNER_SQL );
+ switch( iCol ){
+ case IIC_ERRMSG: {
+ const char *zVal = 0;
+ if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){
+ if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){
+ zVal = "row data mismatch";
+ }
+ }else{
+ zVal = "row missing";
+ }
+ sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC);
+ break;
+ }
+ case IIC_CURRENT_KEY: {
+ sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1));
+ break;
+ }
+ case IIC_INDEX_NAME: {
+ sqlite3_result_text(ctx, pCsr->zIdxName, -1, SQLITE_TRANSIENT);
+ break;
+ }
+ case IIC_AFTER_KEY: {
+ sqlite3_result_text(ctx, pCsr->zAfterKey, -1, SQLITE_TRANSIENT);
+ break;
+ }
+ case IIC_SCANNER_SQL: {
+ char *zSql = 0;
+ cidxGenerateScanSql(pCsr, pCsr->zIdxName, pCsr->zAfterKey, &zSql);
+ sqlite3_result_text(ctx, zSql, -1, sqlite3_free);
+ break;
+ }
+ }
+ return SQLITE_OK;
+}
+
+/* Return the ROWID for the sqlite_btreeinfo table */
+static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+ CidxCursor *pCsr = (CidxCursor*)pCursor;
+ *pRowid = pCsr->iRowid;
+ return SQLITE_OK;
+}
+
+/*
+** Register the virtual table modules with the database handle passed
+** as the only argument.
+*/
+static int ciInit(sqlite3 *db){
+ static sqlite3_module cidx_module = {
+ 0, /* iVersion */
+ 0, /* xCreate */
+ cidxConnect, /* xConnect */
+ cidxBestIndex, /* xBestIndex */
+ cidxDisconnect, /* xDisconnect */
+ 0, /* xDestroy */
+ cidxOpen, /* xOpen - open a cursor */
+ cidxClose, /* xClose - close a cursor */
+ cidxFilter, /* xFilter - configure scan constraints */
+ cidxNext, /* xNext - advance a cursor */
+ cidxEof, /* xEof - check for end of scan */
+ cidxColumn, /* xColumn - read data */
+ cidxRowid, /* xRowid - read data */
+ 0, /* xUpdate */
+ 0, /* xBegin */
+ 0, /* xSync */
+ 0, /* xCommit */
+ 0, /* xRollback */
+ 0, /* xFindMethod */
+ 0, /* xRename */
+ 0, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ };
+ return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0);
+}
+
+/*
+** Extension load function.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_checkindex_init(
+ sqlite3 *db,
+ char **pzErrMsg,
+ const sqlite3_api_routines *pApi
+){
+ SQLITE_EXTENSION_INIT2(pApi);
+ return ciInit(db);
+}
ADDED ext/repair/sqlite3_checker.c.in
Index: ext/repair/sqlite3_checker.c.in
==================================================================
--- /dev/null
+++ ext/repair/sqlite3_checker.c.in
@@ -0,0 +1,86 @@
+/*
+** Read an SQLite database file and analyze its space utilization. Generate
+** text on standard output.
+*/
+#define TCLSH_INIT_PROC sqlite3_checker_init_proc
+#define SQLITE_ENABLE_DBPAGE_VTAB 1
+#define SQLITE_ENABLE_JSON1 1
+#undef SQLITE_THREADSAFE
+#define SQLITE_THREADSAFE 0
+#undef SQLITE_ENABLE_COLUMN_METADATA
+#define SQLITE_OMIT_DECLTYPE 1
+#define SQLITE_OMIT_DEPRECATED 1
+#define SQLITE_OMIT_PROGRESS_CALLBACK 1
+#define SQLITE_OMIT_SHARED_CACHE 1
+#define SQLITE_DEFAULT_MEMSTATUS 0
+#define SQLITE_MAX_EXPR_DEPTH 0
+INCLUDE sqlite3.c
+INCLUDE $ROOT/src/tclsqlite.c
+INCLUDE $ROOT/ext/misc/btreeinfo.c
+INCLUDE $ROOT/ext/repair/checkindex.c
+INCLUDE $ROOT/ext/repair/checkfreelist.c
+
+/*
+** Decode a pointer to an sqlite3 object.
+*/
+int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){
+ struct SqliteDb *p;
+ Tcl_CmdInfo cmdInfo;
+ if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){
+ p = (struct SqliteDb*)cmdInfo.objClientData;
+ *ppDb = p->db;
+ return TCL_OK;
+ }else{
+ *ppDb = 0;
+ return TCL_ERROR;
+ }
+ return TCL_OK;
+}
+
+/*
+** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter
+** sqlite3_imposter db main ;# rm all imposters
+*/
+static int sqlite3_imposter(
+ void *clientData,
+ Tcl_Interp *interp,
+ int objc,
+ Tcl_Obj *CONST objv[]
+){
+ sqlite3 *db;
+ const char *zSchema;
+ int iRoot;
+ const char *zSql;
+
+ if( objc!=3 && objc!=5 ){
+ Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]");
+ return TCL_ERROR;
+ }
+ if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR;
+ zSchema = Tcl_GetString(objv[2]);
+ if( objc==3 ){
+ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1);
+ }else{
+ if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR;
+ zSql = Tcl_GetString(objv[4]);
+ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot);
+ sqlite3_exec(db, zSql, 0, 0, 0);
+ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0);
+ }
+ return TCL_OK;
+}
+
+#include
+
+const char *sqlite3_checker_init_proc(Tcl_Interp *interp){
+ Tcl_CreateObjCommand(interp, "sqlite3_imposter",
+ (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0);
+ sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init);
+ sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init);
+ sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init);
+ return
+BEGIN_STRING
+INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl
+END_STRING
+;
+}
ADDED ext/repair/sqlite3_checker.tcl
Index: ext/repair/sqlite3_checker.tcl
==================================================================
--- /dev/null
+++ ext/repair/sqlite3_checker.tcl
@@ -0,0 +1,264 @@
+# This TCL script is the main driver script for the sqlite3_checker utility
+# program.
+#
+
+# Special case:
+#
+# sqlite3_checker --test FILENAME ARGS
+#
+# uses FILENAME in place of this script.
+#
+if {[lindex $argv 0]=="--test" && [llength $argv]>1} {
+ set ::argv0 [lindex $argv 1]
+ set argv [lrange $argv 2 end]
+ source $argv0
+ exit 0
+}
+
+# Emulate a TCL shell
+#
+proc tclsh {} {
+ set line {}
+ while {![eof stdin]} {
+ if {$line!=""} {
+ puts -nonewline "> "
+ } else {
+ puts -nonewline "% "
+ }
+ flush stdout
+ append line [gets stdin]
+ if {[info complete $line]} {
+ if {[catch {uplevel #0 $line} result]} {
+ puts stderr "Error: $result"
+ } elseif {$result!=""} {
+ puts $result
+ }
+ set line {}
+ } else {
+ append line \n
+ }
+ }
+}
+
+# Do an incremental integrity check of a single index
+#
+proc check_index {idxname batchsize bTrace} {
+ set i 0
+ set more 1
+ set nerr 0
+ set pct 00.0
+ set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main')
+ WHERE name=$idxname}]
+ puts -nonewline "$idxname: $i of $max rows ($pct%)\r"
+ flush stdout
+ if {$bTrace} {
+ set sql {SELECT errmsg, current_key AS key,
+ CASE WHEN rowid=1 THEN scanner_sql END AS traceOut
+ FROM incremental_index_check($idxname)
+ WHERE after_key=$key
+ LIMIT $batchsize}
+ } else {
+ set sql {SELECT errmsg, current_key AS key, NULL AS traceOut
+ FROM incremental_index_check($idxname)
+ WHERE after_key=$key
+ LIMIT $batchsize}
+ }
+ while {$more} {
+ set more 0
+ db eval $sql {
+ set more 1
+ if {$errmsg!=""} {
+ incr nerr
+ puts "$idxname: key($key): $errmsg"
+ } elseif {$traceOut!=""} {
+ puts "$idxname: $traceOut"
+ }
+ incr i
+
+ }
+ set x [format {%.1f} [expr {($i*100.0)/$max}]]
+ if {$x!=$pct} {
+ puts -nonewline "$idxname: $i of $max rows ($pct%)\r"
+ flush stdout
+ set pct $x
+ }
+ }
+ puts "$idxname: $nerr errors out of $i entries"
+}
+
+# Print a usage message on standard error, then quit.
+#
+proc usage {} {
+ set argv0 [file rootname [file tail [info nameofexecutable]]]
+ puts stderr "Usage: $argv0 OPTIONS database-filename"
+ puts stderr {
+Do sanity checking on a live SQLite3 database file specified by the
+"database-filename" argument.
+
+Options:
+
+ --batchsize N Number of rows to check per transaction
+
+ --freelist Perform a freelist check
+
+ --index NAME Run a check of the index NAME
+
+ --summary Print summary information about the database
+
+ --table NAME Run a check of all indexes for table NAME
+
+ --tclsh Run the built-in TCL interpreter (for debugging)
+
+ --trace (Debugging only:) Output trace information on the scan
+
+ --version Show the version number of SQLite
+}
+ exit 1
+}
+
+set file_to_analyze {}
+append argv {}
+set bFreelistCheck 0
+set bSummary 0
+set zIndex {}
+set zTable {}
+set batchsize 1000
+set bAll 1
+set bTrace 0
+set argc [llength $argv]
+for {set i 0} {$i<$argc} {incr i} {
+ set arg [lindex $argv $i]
+ if {[regexp {^-+tclsh$} $arg]} {
+ tclsh
+ exit 0
+ }
+ if {[regexp {^-+version$} $arg]} {
+ sqlite3 mem :memory:
+ puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}]
+ mem close
+ exit 0
+ }
+ if {[regexp {^-+freelist$} $arg]} {
+ set bFreelistCheck 1
+ set bAll 0
+ continue
+ }
+ if {[regexp {^-+summary$} $arg]} {
+ set bSummary 1
+ set bAll 0
+ continue
+ }
+ if {[regexp {^-+trace$} $arg]} {
+ set bTrace 1
+ continue
+ }
+ if {[regexp {^-+batchsize$} $arg]} {
+ incr i
+ if {$i>=$argc} {
+ puts stderr "missing argument on $arg"
+ exit 1
+ }
+ set batchsize [lindex $argv $i]
+ continue
+ }
+ if {[regexp {^-+index$} $arg]} {
+ incr i
+ if {$i>=$argc} {
+ puts stderr "missing argument on $arg"
+ exit 1
+ }
+ set zIndex [lindex $argv $i]
+ set bAll 0
+ continue
+ }
+ if {[regexp {^-+table$} $arg]} {
+ incr i
+ if {$i>=$argc} {
+ puts stderr "missing argument on $arg"
+ exit 1
+ }
+ set zTable [lindex $argv $i]
+ set bAll 0
+ continue
+ }
+ if {[regexp {^-} $arg]} {
+ puts stderr "Unknown option: $arg"
+ usage
+ }
+ if {$file_to_analyze!=""} {
+ usage
+ } else {
+ set file_to_analyze $arg
+ }
+}
+if {$file_to_analyze==""} usage
+
+# If a TCL script is specified on the command-line, then run that
+# script.
+#
+if {[file extension $file_to_analyze]==".tcl"} {
+ source $file_to_analyze
+ exit 0
+}
+
+set root_filename $file_to_analyze
+regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename
+if {![file exists $root_filename]} {
+ puts stderr "No such file: $root_filename"
+ exit 1
+}
+if {![file readable $root_filename]} {
+ puts stderr "File is not readable: $root_filename"
+ exit 1
+}
+
+if {[catch {sqlite3 db $file_to_analyze} res]} {
+ puts stderr "Cannot open datababase $root_filename: $res"
+ exit 1
+}
+
+if {$bFreelistCheck || $bAll} {
+ puts -nonewline "freelist-check: "
+ flush stdout
+ db eval BEGIN
+ puts [db one {SELECT checkfreelist('main')}]
+ db eval END
+}
+if {$bSummary} {
+ set scale 0
+ set pgsz [db one {PRAGMA page_size}]
+ db eval {SELECT nPage*$pgsz AS sz, name, tbl_name
+ FROM sqlite_btreeinfo
+ WHERE type='index'
+ ORDER BY 1 DESC, name} {
+ if {$scale==0} {
+ if {$sz>10000000} {
+ set scale 1000000.0
+ set unit MB
+ } else {
+ set scale 1000.0
+ set unit KB
+ }
+ }
+ puts [format {%7.1f %s index %s of table %s} \
+ [expr {$sz/$scale}] $unit $name $tbl_name]
+ }
+}
+if {$zIndex!=""} {
+ check_index $zIndex $batchsize $bTrace
+}
+if {$zTable!=""} {
+ foreach idx [db eval {SELECT name FROM sqlite_master
+ WHERE type='index' AND rootpage>0
+ AND tbl_name=$zTable}] {
+ check_index $idx $batchsize $bTrace
+ }
+}
+if {$bAll} {
+ set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main')
+ WHERE type='index' AND rootpage>0
+ ORDER BY nEntry}]
+ foreach idx $allidx {
+ check_index $idx $batchsize $bTrace
+ }
+}
ADDED ext/repair/test/README.md
Index: ext/repair/test/README.md
==================================================================
--- /dev/null
+++ ext/repair/test/README.md
@@ -0,0 +1,13 @@
+To run these tests, first build sqlite3_checker:
+
+
+> make sqlite3_checker
+
+
+Then run the "test.tcl" script using:
+
+
+> ./sqlite3_checker --test $path/test.tcl
+
+
+Optionally add the full pathnames of individual *.test modules
ADDED ext/repair/test/checkfreelist01.test
Index: ext/repair/test/checkfreelist01.test
==================================================================
--- /dev/null
+++ ext/repair/test/checkfreelist01.test
@@ -0,0 +1,92 @@
+# 2017-10-11
+
+set testprefix checkfreelist
+
+do_execsql_test 1.0 {
+ PRAGMA page_size=1024;
+ CREATE TABLE t1(a, b);
+}
+
+do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok}
+do_execsql_test 1.3 {
+ WITH s(i) AS (
+ SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10000
+ )
+ INSERT INTO t1 SELECT randomblob(400), randomblob(400) FROM s;
+ DELETE FROM t1 WHERE rowid%3;
+ PRAGMA freelist_count;
+} {6726}
+
+do_execsql_test 1.4 { SELECT checkfreelist('main') } {ok}
+do_execsql_test 1.5 {
+ WITH freelist_trunk(i, d, n) AS (
+ SELECT 1, NULL, sqlite_readint32(data, 32) FROM sqlite_dbpage WHERE pgno=1
+ UNION ALL
+ SELECT n, data, sqlite_readint32(data)
+ FROM freelist_trunk, sqlite_dbpage WHERE pgno=n
+ )
+ SELECT i FROM freelist_trunk WHERE i!=1;
+} {
+ 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234
+ 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5
+}
+
+do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok}
+
+proc set_int {blob idx newval} {
+ binary scan $blob I* ints
+ lset ints $idx $newval
+ binary format I* $ints
+}
+db func set_int set_int
+
+proc get_int {blob idx} {
+ binary scan $blob I* ints
+ lindex $ints $idx
+}
+db func get_int get_int
+
+do_execsql_test 1.7 {
+ BEGIN;
+ UPDATE sqlite_dbpage
+ SET data = set_int(data, 1, get_int(data, 1)-1)
+ WHERE pgno=4860;
+ SELECT checkfreelist('main');
+ ROLLBACK;
+} {{free-list count mismatch: actual=6725 header=6726}}
+
+do_execsql_test 1.8 {
+ BEGIN;
+ UPDATE sqlite_dbpage
+ SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1)
+ WHERE pgno=4860;
+ SELECT checkfreelist('main');
+ ROLLBACK;
+} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}}
+
+do_execsql_test 1.9 {
+ BEGIN;
+ UPDATE sqlite_dbpage
+ SET data = set_int(data, 5, 0)
+ WHERE pgno=4860;
+ SELECT checkfreelist('main');
+ ROLLBACK;
+} {{leaf page 0 is out of range (child 3 of trunk page 4860)}}
+
+do_execsql_test 1.10 {
+ BEGIN;
+ UPDATE sqlite_dbpage
+ SET data = set_int(data, get_int(data, 1)+1, 0)
+ WHERE pgno=5;
+ SELECT checkfreelist('main');
+ ROLLBACK;
+} {{leaf page 0 is out of range (child 247 of trunk page 5)}}
+
+do_execsql_test 1.11 {
+ BEGIN;
+ UPDATE sqlite_dbpage
+ SET data = set_int(data, 1, 249)
+ WHERE pgno=5;
+ SELECT checkfreelist('main');
+ ROLLBACK;
+} {{leaf count out of range (249) on trunk page 5}}
ADDED ext/repair/test/checkindex01.test
Index: ext/repair/test/checkindex01.test
==================================================================
--- /dev/null
+++ ext/repair/test/checkindex01.test
@@ -0,0 +1,349 @@
+# 2017-10-11
+#
+set testprefix checkindex
+
+do_execsql_test 1.0 {
+ CREATE TABLE t1(a, b);
+ CREATE INDEX i1 ON t1(a);
+ INSERT INTO t1 VALUES('one', 2);
+ INSERT INTO t1 VALUES('two', 4);
+ INSERT INTO t1 VALUES('three', 6);
+ INSERT INTO t1 VALUES('four', 8);
+ INSERT INTO t1 VALUES('five', 10);
+
+ CREATE INDEX i2 ON t1(a DESC);
+} {}
+
+proc incr_index_check {idx nStep} {
+ set Q {
+ SELECT errmsg, current_key FROM incremental_index_check($idx, $after)
+ LIMIT $nStep
+ }
+
+ set res [list]
+ while {1} {
+ unset -nocomplain current_key
+ set res1 [db eval $Q]
+ if {[llength $res1]==0} break
+ set res [concat $res $res1]
+ set after [lindex $res end]
+ }
+
+ return $res
+}
+
+proc do_index_check_test {tn idx res} {
+ uplevel [list do_execsql_test $tn.1 "
+ SELECT errmsg, current_key FROM incremental_index_check('$idx');
+ " $res]
+
+ uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]]
+ uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]]
+ uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]]
+}
+
+
+do_execsql_test 1.2.1 {
+ SELECT rowid, errmsg IS NULL, current_key FROM incremental_index_check('i1');
+} {
+ 1 1 'five',5
+ 2 1 'four',4
+ 3 1 'one',1
+ 4 1 'three',3
+ 5 1 'two',2
+}
+do_execsql_test 1.2.2 {
+ SELECT errmsg IS NULL, current_key, index_name, after_key, scanner_sql
+ FROM incremental_index_check('i1') LIMIT 1;
+} {
+ 1
+ 'five',5
+ i1
+ {}
+ {SELECT (SELECT a IS i.i0 FROM 't1' AS t WHERE "rowid" COLLATE BINARY IS i.i1), quote(i0)||','||quote(i1) FROM (SELECT (a) AS i0, ("rowid" COLLATE BINARY) AS i1 FROM 't1' INDEXED BY 'i1' ORDER BY 1,2) AS i}
+}
+
+do_index_check_test 1.3 i1 {
+ {} 'five',5
+ {} 'four',4
+ {} 'one',1
+ {} 'three',3
+ {} 'two',2
+}
+
+do_index_check_test 1.4 i2 {
+ {} 'two',2
+ {} 'three',3
+ {} 'one',1
+ {} 'four',4
+ {} 'five',5
+}
+
+do_test 1.5 {
+ set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }]
+ sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)}
+ db eval {
+ UPDATE xt1 SET a='six' WHERE rowid=3;
+ DELETE FROM xt1 WHERE rowid = 5;
+ }
+ sqlite3_imposter db main
+} {}
+
+do_index_check_test 1.6 i1 {
+ {row missing} 'five',5
+ {} 'four',4
+ {} 'one',1
+ {row data mismatch} 'three',3
+ {} 'two',2
+}
+
+do_index_check_test 1.7 i2 {
+ {} 'two',2
+ {row data mismatch} 'three',3
+ {} 'one',1
+ {} 'four',4
+ {row missing} 'five',5
+}
+
+#--------------------------------------------------------------------------
+do_execsql_test 2.0 {
+
+ CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d);
+
+ INSERT INTO t2 VALUES(1, NULL, 1, 1);
+ INSERT INTO t2 VALUES(2, 1, NULL, 1);
+ INSERT INTO t2 VALUES(3, 1, 1, NULL);
+
+ INSERT INTO t2 VALUES(4, 2, 2, 1);
+ INSERT INTO t2 VALUES(5, 2, 2, 2);
+ INSERT INTO t2 VALUES(6, 2, 2, 3);
+
+ INSERT INTO t2 VALUES(7, 2, 2, 1);
+ INSERT INTO t2 VALUES(8, 2, 2, 2);
+ INSERT INTO t2 VALUES(9, 2, 2, 3);
+
+ CREATE INDEX i3 ON t2(b, c, d);
+ CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC);
+ CREATE INDEX i5 ON t2(d, c DESC, b);
+} {}
+
+do_index_check_test 2.1 i3 {
+ {} NULL,1,1,1
+ {} 1,NULL,1,2
+ {} 1,1,NULL,3
+ {} 2,2,1,4
+ {} 2,2,1,7
+ {} 2,2,2,5
+ {} 2,2,2,8
+ {} 2,2,3,6
+ {} 2,2,3,9
+}
+
+do_index_check_test 2.2 i4 {
+ {} 2,2,3,6
+ {} 2,2,3,9
+ {} 2,2,2,5
+ {} 2,2,2,8
+ {} 2,2,1,4
+ {} 2,2,1,7
+ {} 1,1,NULL,3
+ {} 1,NULL,1,2
+ {} NULL,1,1,1
+}
+
+do_index_check_test 2.3 i5 {
+ {} NULL,1,1,3
+ {} 1,2,2,4
+ {} 1,2,2,7
+ {} 1,1,NULL,1
+ {} 1,NULL,1,2
+ {} 2,2,2,5
+ {} 2,2,2,8
+ {} 3,2,2,6
+ {} 3,2,2,9
+}
+
+#--------------------------------------------------------------------------
+do_execsql_test 3.0 {
+
+ CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID;
+ CREATE INDEX t3wxy ON t3(w, x, y);
+ CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC);
+
+ INSERT INTO t3 VALUES(NULL, NULL, NULL, 1);
+ INSERT INTO t3 VALUES(NULL, NULL, NULL, 2);
+ INSERT INTO t3 VALUES(NULL, NULL, NULL, 3);
+
+ INSERT INTO t3 VALUES('a', NULL, NULL, 4);
+ INSERT INTO t3 VALUES('a', NULL, NULL, 5);
+ INSERT INTO t3 VALUES('a', NULL, NULL, 6);
+
+ INSERT INTO t3 VALUES('a', 'b', NULL, 7);
+ INSERT INTO t3 VALUES('a', 'b', NULL, 8);
+ INSERT INTO t3 VALUES('a', 'b', NULL, 9);
+
+} {}
+
+do_index_check_test 3.1 t3wxy {
+ {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3
+ {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6
+ {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9
+}
+do_index_check_test 3.2 t3wxy2 {
+ {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9
+ {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6
+ {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3
+}
+
+#--------------------------------------------------------------------------
+# Test with an index that uses non-default collation sequences.
+#
+do_execsql_test 4.0 {
+ CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT);
+ INSERT INTO t4 VALUES(1, 'aaa', 'bbb');
+ INSERT INTO t4 VALUES(2, 'AAA', 'CCC');
+ INSERT INTO t4 VALUES(3, 'aab', 'ddd');
+ INSERT INTO t4 VALUES(4, 'AAB', 'EEE');
+
+ CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase);
+}
+
+do_index_check_test 4.1 t4cc {
+ {} 'aaa','bbb',1
+ {} 'AAA','CCC',2
+ {} 'aab','ddd',3
+ {} 'AAB','EEE',4
+}
+
+do_test 4.2 {
+ set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }]
+ sqlite3_imposter db main $tblroot \
+ {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)}
+
+ db eval {
+ UPDATE xt4 SET c1='hello' WHERE rowid=2;
+ DELETE FROM xt4 WHERE rowid = 3;
+ }
+ sqlite3_imposter db main
+} {}
+
+do_index_check_test 4.3 t4cc {
+ {} 'aaa','bbb',1
+ {row data mismatch} 'AAA','CCC',2
+ {row missing} 'aab','ddd',3
+ {} 'AAB','EEE',4
+}
+
+#--------------------------------------------------------------------------
+# Test an index on an expression.
+#
+do_execsql_test 5.0 {
+ CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y));
+ INSERT INTO t5 VALUES(1, '{"x":1, "y":1}');
+ INSERT INTO t5 VALUES(2, '{"x":2, "y":2}');
+ INSERT INTO t5 VALUES(3, '{"x":3, "y":3}');
+ INSERT INTO t5 VALUES(4, '{"w":4, "z":4}');
+ INSERT INTO t5 VALUES(5, '{"x":5, "y":5}');
+
+ CREATE INDEX t5x ON t5( json_extract(y, '$.x') );
+ CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC );
+}
+
+do_index_check_test 5.1.1 t5x {
+ {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5
+}
+
+do_index_check_test 5.1.2 t5y {
+ {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4
+}
+
+do_index_check_test 5.1.3 sqlite_autoindex_t5_1 {
+ {} {'{"w":4, "z":4}',4}
+ {} {'{"x":1, "y":1}',1}
+ {} {'{"x":2, "y":2}',2}
+ {} {'{"x":3, "y":3}',3}
+ {} {'{"x":5, "y":5}',5}
+}
+
+do_test 5.2 {
+ set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }]
+ sqlite3_imposter db main $tblroot \
+ {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);}
+ db eval {
+ UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1;
+ DELETE FROM xt5 WHERE rowid = 4;
+ }
+ sqlite3_imposter db main
+} {}
+
+do_index_check_test 5.3.1 t5x {
+ {row missing} NULL,4
+ {row data mismatch} 1,1
+ {} 2,2
+ {} 3,3
+ {} 5,5
+}
+
+do_index_check_test 5.3.2 sqlite_autoindex_t5_1 {
+ {row missing} {'{"w":4, "z":4}',4}
+ {row data mismatch} {'{"x":1, "y":1}',1}
+ {} {'{"x":2, "y":2}',2}
+ {} {'{"x":3, "y":3}',3}
+ {} {'{"x":5, "y":5}',5}
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 6.0 {
+ CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z);
+ CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z);
+ CREATE INDEX t6x2 ON t6(z, -- hello,world,
+ y);
+
+ CREATE INDEX t6x3 ON t6(z -- hello,world
+ , y);
+
+ INSERT INTO t6 VALUES(1, 2, 3);
+ INSERT INTO t6 VALUES(4, 5, 6);
+}
+
+do_index_check_test 6.1 t6x1 {
+ {} 2,3,1
+ {} 5,6,4
+}
+do_index_check_test 6.2 t6x2 {
+ {} 3,2,1
+ {} 6,5,4
+}
+do_index_check_test 6.2 t6x3 {
+ {} 3,2,1
+ {} 6,5,4
+}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 7.0 {
+ CREATE TABLE t7(x INTEGER PRIMARY KEY, y, z);
+ INSERT INTO t7 VALUES(1, 1, 1);
+ INSERT INTO t7 VALUES(2, 2, 0);
+ INSERT INTO t7 VALUES(3, 3, 1);
+ INSERT INTO t7 VALUES(4, 4, 0);
+
+ CREATE INDEX t7i1 ON t7(y) WHERE z=1;
+ CREATE INDEX t7i2 ON t7(y) /* hello,world */ WHERE z=1;
+ CREATE INDEX t7i3 ON t7(y) WHERE -- yep
+ z=1;
+ CREATE INDEX t7i4 ON t7(y) WHERE z=1 -- yep;
+}
+do_index_check_test 7.1 t7i1 {
+ {} 1,1 {} 3,3
+}
+do_index_check_test 7.2 t7i2 {
+ {} 1,1 {} 3,3
+}
+do_index_check_test 7.3 t7i3 {
+ {} 1,1 {} 3,3
+}
+do_index_check_test 7.4 t7i4 {
+ {} 1,1 {} 3,3
+}
ADDED ext/repair/test/test.tcl
Index: ext/repair/test/test.tcl
==================================================================
--- /dev/null
+++ ext/repair/test/test.tcl
@@ -0,0 +1,67 @@
+# Run this script using
+#
+# sqlite3_checker --test $thisscript $testscripts
+#
+# The $testscripts argument is optional. If omitted, all *.test files
+# in the same directory as $thisscript are run.
+#
+set NTEST 0
+set NERR 0
+
+
+# Invoke the do_test procedure to run a single test
+#
+# The $expected parameter is the expected result. The result is the return
+# value from the last TCL command in $cmd.
+#
+# Normally, $expected must match exactly. But if $expected is of the form
+# "/regexp/" then regular expression matching is used. If $expected is
+# "~/regexp/" then the regular expression must NOT match. If $expected is
+# of the form "#/value-list/" then each term in value-list must be numeric
+# and must approximately match the corresponding numeric term in $result.
+# Values must match within 10%. Or if the $expected term is A..B then the
+# $result term must be in between A and B.
+#
+proc do_test {name cmd expected} {
+ if {[info exists ::testprefix]} {
+ set name "$::testprefix$name"
+ }
+
+ incr ::NTEST
+ puts -nonewline $name...
+ flush stdout
+
+ if {[catch {uplevel #0 "$cmd;\n"} result]} {
+ puts -nonewline $name...
+ puts "\nError: $result"
+ incr ::NERR
+ } else {
+ set ok [expr {[string compare $result $expected]==0}]
+ if {!$ok} {
+ puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]"
+ incr ::NERR
+ } else {
+ puts " Ok"
+ }
+ }
+ flush stdout
+}
+
+#
+# do_execsql_test TESTNAME SQL RES
+#
+proc do_execsql_test {testname sql {result {}}} {
+ uplevel [list do_test $testname [list db eval $sql] [list {*}$result]]
+}
+
+if {[llength $argv]==0} {
+ set dir [file dirname $argv0]
+ set argv [glob -nocomplain $dir/*.test]
+}
+foreach testfile $argv {
+ file delete -force test.db
+ sqlite3 db test.db
+ source $testfile
+ catch {db close}
+}
+puts "$NERR errors out of $NTEST tests"
ADDED ext/rtree/geopoly.c
Index: ext/rtree/geopoly.c
==================================================================
--- /dev/null
+++ ext/rtree/geopoly.c
@@ -0,0 +1,1797 @@
+/*
+** 2018-05-25
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file implements an alternative R-Tree virtual table that
+** uses polygons to express the boundaries of 2-dimensional objects.
+**
+** This file is #include-ed onto the end of "rtree.c" so that it has
+** access to all of the R-Tree internals.
+*/
+#include
+
+/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */
+#ifdef GEOPOLY_ENABLE_DEBUG
+ static int geo_debug = 0;
+# define GEODEBUG(X) if(geo_debug)printf X
+#else
+# define GEODEBUG(X)
+#endif
+
+#ifndef JSON_NULL /* The following stuff repeats things found in json1 */
+/*
+** Versions of isspace(), isalnum() and isdigit() to which it is safe
+** to pass signed char values.
+*/
+#ifdef sqlite3Isdigit
+ /* Use the SQLite core versions if this routine is part of the
+ ** SQLite amalgamation */
+# define safe_isdigit(x) sqlite3Isdigit(x)
+# define safe_isalnum(x) sqlite3Isalnum(x)
+# define safe_isxdigit(x) sqlite3Isxdigit(x)
+#else
+ /* Use the standard library for separate compilation */
+#include /* amalgamator: keep */
+# define safe_isdigit(x) isdigit((unsigned char)(x))
+# define safe_isalnum(x) isalnum((unsigned char)(x))
+# define safe_isxdigit(x) isxdigit((unsigned char)(x))
+#endif
+
+/*
+** Growing our own isspace() routine this way is twice as fast as
+** the library isspace() function.
+*/
+static const char geopolyIsSpace[] = {
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+};
+#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x])
+#endif /* JSON NULL - back to original code */
+
+/* Compiler and version */
+#ifndef GCC_VERSION
+#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC)
+# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__)
+#else
+# define GCC_VERSION 0
+#endif
+#endif
+#ifndef MSVC_VERSION
+#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC)
+# define MSVC_VERSION _MSC_VER
+#else
+# define MSVC_VERSION 0
+#endif
+#endif
+
+/* Datatype for coordinates
+*/
+typedef float GeoCoord;
+
+/*
+** Internal representation of a polygon.
+**
+** The polygon consists of a sequence of vertexes. There is a line
+** segment between each pair of vertexes, and one final segment from
+** the last vertex back to the first. (This differs from the GeoJSON
+** standard in which the final vertex is a repeat of the first.)
+**
+** The polygon follows the right-hand rule. The area to the right of
+** each segment is "outside" and the area to the left is "inside".
+**
+** The on-disk representation consists of a 4-byte header followed by
+** the values. The 4-byte header is:
+**
+** encoding (1 byte) 0=big-endian, 1=little-endian
+** nvertex (3 bytes) Number of vertexes as a big-endian integer
+**
+** Enough space is allocated for 4 coordinates, to work around over-zealous
+** warnings coming from some compiler (notably, clang). In reality, the size
+** of each GeoPoly memory allocate is adjusted as necessary so that the
+** GeoPoly.a[] array at the end is the appropriate size.
+*/
+typedef struct GeoPoly GeoPoly;
+struct GeoPoly {
+ int nVertex; /* Number of vertexes */
+ unsigned char hdr[4]; /* Header for on-disk representation */
+ GeoCoord a[8]; /* 2*nVertex values. X (longitude) first, then Y */
+};
+
+/* The size of a memory allocation needed for a GeoPoly object sufficient
+** to hold N coordinate pairs.
+*/
+#define GEOPOLY_SZ(N) (sizeof(GeoPoly) + sizeof(GeoCoord)*2*((N)-4))
+
+/*
+** State of a parse of a GeoJSON input.
+*/
+typedef struct GeoParse GeoParse;
+struct GeoParse {
+ const unsigned char *z; /* Unparsed input */
+ int nVertex; /* Number of vertexes in a[] */
+ int nAlloc; /* Space allocated to a[] */
+ int nErr; /* Number of errors encountered */
+ GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */
+};
+
+/* Do a 4-byte byte swap */
+static void geopolySwab32(unsigned char *a){
+ unsigned char t = a[0];
+ a[0] = a[3];
+ a[3] = t;
+ t = a[1];
+ a[1] = a[2];
+ a[2] = t;
+}
+
+/* Skip whitespace. Return the next non-whitespace character. */
+static char geopolySkipSpace(GeoParse *p){
+ while( safe_isspace(p->z[0]) ) p->z++;
+ return p->z[0];
+}
+
+/* Parse out a number. Write the value into *pVal if pVal!=0.
+** return non-zero on success and zero if the next token is not a number.
+*/
+static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){
+ char c = geopolySkipSpace(p);
+ const unsigned char *z = p->z;
+ int j = 0;
+ int seenDP = 0;
+ int seenE = 0;
+ if( c=='-' ){
+ j = 1;
+ c = z[j];
+ }
+ if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0;
+ for(;; j++){
+ c = z[j];
+ if( safe_isdigit(c) ) continue;
+ if( c=='.' ){
+ if( z[j-1]=='-' ) return 0;
+ if( seenDP ) return 0;
+ seenDP = 1;
+ continue;
+ }
+ if( c=='e' || c=='E' ){
+ if( z[j-1]<'0' ) return 0;
+ if( seenE ) return -1;
+ seenDP = seenE = 1;
+ c = z[j+1];
+ if( c=='+' || c=='-' ){
+ j++;
+ c = z[j+1];
+ }
+ if( c<'0' || c>'9' ) return 0;
+ continue;
+ }
+ break;
+ }
+ if( z[j-1]<'0' ) return 0;
+ if( pVal ){
+#ifdef SQLITE_AMALGAMATION
+ /* The sqlite3AtoF() routine is much much faster than atof(), if it
+ ** is available */
+ double r;
+ (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8);
+ *pVal = r;
+#else
+ *pVal = (GeoCoord)atof((const char*)p->z);
+#endif
+ }
+ p->z += j;
+ return 1;
+}
+
+/*
+** If the input is a well-formed JSON array of coordinates with at least
+** four coordinates and where each coordinate is itself a two-value array,
+** then convert the JSON into a GeoPoly object and return a pointer to
+** that object.
+**
+** If any error occurs, return NULL.
+*/
+static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){
+ GeoParse s;
+ int rc = SQLITE_OK;
+ memset(&s, 0, sizeof(s));
+ s.z = z;
+ if( geopolySkipSpace(&s)=='[' ){
+ s.z++;
+ while( geopolySkipSpace(&s)=='[' ){
+ int ii = 0;
+ char c;
+ s.z++;
+ if( s.nVertex>=s.nAlloc ){
+ GeoCoord *aNew;
+ s.nAlloc = s.nAlloc*2 + 16;
+ aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 );
+ if( aNew==0 ){
+ rc = SQLITE_NOMEM;
+ s.nErr++;
+ break;
+ }
+ s.a = aNew;
+ }
+ while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){
+ ii++;
+ if( ii==2 ) s.nVertex++;
+ c = geopolySkipSpace(&s);
+ s.z++;
+ if( c==',' ) continue;
+ if( c==']' && ii>=2 ) break;
+ s.nErr++;
+ rc = SQLITE_ERROR;
+ goto parse_json_err;
+ }
+ if( geopolySkipSpace(&s)==',' ){
+ s.z++;
+ continue;
+ }
+ break;
+ }
+ if( geopolySkipSpace(&s)==']'
+ && s.nVertex>=4
+ && s.a[0]==s.a[s.nVertex*2-2]
+ && s.a[1]==s.a[s.nVertex*2-1]
+ && (s.z++, geopolySkipSpace(&s)==0)
+ ){
+ GeoPoly *pOut;
+ int x = 1;
+ s.nVertex--; /* Remove the redundant vertex at the end */
+ pOut = sqlite3_malloc64( GEOPOLY_SZ(s.nVertex) );
+ x = 1;
+ if( pOut==0 ) goto parse_json_err;
+ pOut->nVertex = s.nVertex;
+ memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord));
+ pOut->hdr[0] = *(unsigned char*)&x;
+ pOut->hdr[1] = (s.nVertex>>16)&0xff;
+ pOut->hdr[2] = (s.nVertex>>8)&0xff;
+ pOut->hdr[3] = s.nVertex&0xff;
+ sqlite3_free(s.a);
+ if( pRc ) *pRc = SQLITE_OK;
+ return pOut;
+ }else{
+ s.nErr++;
+ rc = SQLITE_ERROR;
+ }
+ }
+parse_json_err:
+ if( pRc ) *pRc = rc;
+ sqlite3_free(s.a);
+ return 0;
+}
+
+/*
+** Given a function parameter, try to interpret it as a polygon, either
+** in the binary format or JSON text. Compute a GeoPoly object and
+** return a pointer to that object. Or if the input is not a well-formed
+** polygon, put an error message in sqlite3_context and return NULL.
+*/
+static GeoPoly *geopolyFuncParam(
+ sqlite3_context *pCtx, /* Context for error messages */
+ sqlite3_value *pVal, /* The value to decode */
+ int *pRc /* Write error here */
+){
+ GeoPoly *p = 0;
+ int nByte;
+ if( sqlite3_value_type(pVal)==SQLITE_BLOB
+ && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord))
+ ){
+ const unsigned char *a = sqlite3_value_blob(pVal);
+ int nVertex;
+ nVertex = (a[1]<<16) + (a[2]<<8) + a[3];
+ if( (a[0]==0 || a[0]==1)
+ && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte
+ ){
+ p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) );
+ if( p==0 ){
+ if( pRc ) *pRc = SQLITE_NOMEM;
+ if( pCtx ) sqlite3_result_error_nomem(pCtx);
+ }else{
+ int x = 1;
+ p->nVertex = nVertex;
+ memcpy(p->hdr, a, nByte);
+ if( a[0] != *(unsigned char*)&x ){
+ int ii;
+ for(ii=0; iia[ii]);
+ }
+ p->hdr[0] ^= 1;
+ }
+ }
+ }
+ if( pRc ) *pRc = SQLITE_OK;
+ return p;
+ }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){
+ const unsigned char *zJson = sqlite3_value_text(pVal);
+ if( zJson==0 ){
+ if( pRc ) *pRc = SQLITE_NOMEM;
+ return 0;
+ }
+ return geopolyParseJson(zJson, pRc);
+ }else{
+ if( pRc ) *pRc = SQLITE_ERROR;
+ return 0;
+ }
+}
+
+/*
+** Implementation of the geopoly_blob(X) function.
+**
+** If the input is a well-formed Geopoly BLOB or JSON string
+** then return the BLOB representation of the polygon. Otherwise
+** return NULL.
+*/
+static void geopolyBlobFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ if( p ){
+ sqlite3_result_blob(context, p->hdr,
+ 4+8*p->nVertex, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** SQL function: geopoly_json(X)
+**
+** Interpret X as a polygon and render it as a JSON array
+** of coordinates. Or, if X is not a valid polygon, return NULL.
+*/
+static void geopolyJsonFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ if( p ){
+ sqlite3 *db = sqlite3_context_db_handle(context);
+ sqlite3_str *x = sqlite3_str_new(db);
+ int i;
+ sqlite3_str_append(x, "[", 1);
+ for(i=0; inVertex; i++){
+ sqlite3_str_appendf(x, "[%!g,%!g],", p->a[i*2], p->a[i*2+1]);
+ }
+ sqlite3_str_appendf(x, "[%!g,%!g]]", p->a[0], p->a[1]);
+ sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** SQL function: geopoly_svg(X, ....)
+**
+** Interpret X as a polygon and render it as a SVG .
+** Additional arguments are added as attributes to the .
+*/
+static void geopolySvgFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ if( p ){
+ sqlite3 *db = sqlite3_context_db_handle(context);
+ sqlite3_str *x = sqlite3_str_new(db);
+ int i;
+ char cSep = '\'';
+ sqlite3_str_appendf(x, "a[i*2], p->a[i*2+1]);
+ cSep = ' ';
+ }
+ sqlite3_str_appendf(x, " %g,%g'", p->a[0], p->a[1]);
+ for(i=1; i");
+ sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** SQL Function: geopoly_xform(poly, A, B, C, D, E, F)
+**
+** Transform and/or translate a polygon as follows:
+**
+** x1 = A*x0 + B*y0 + E
+** y1 = C*x0 + D*y0 + F
+**
+** For a translation:
+**
+** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset)
+**
+** Rotate by R around the point (0,0):
+**
+** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0)
+*/
+static void geopolyXformFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ double A = sqlite3_value_double(argv[1]);
+ double B = sqlite3_value_double(argv[2]);
+ double C = sqlite3_value_double(argv[3]);
+ double D = sqlite3_value_double(argv[4]);
+ double E = sqlite3_value_double(argv[5]);
+ double F = sqlite3_value_double(argv[6]);
+ GeoCoord x1, y1, x0, y0;
+ int ii;
+ if( p ){
+ for(ii=0; iinVertex; ii++){
+ x0 = p->a[ii*2];
+ y0 = p->a[ii*2+1];
+ x1 = (GeoCoord)(A*x0 + B*y0 + E);
+ y1 = (GeoCoord)(C*x0 + D*y0 + F);
+ p->a[ii*2] = x1;
+ p->a[ii*2+1] = y1;
+ }
+ sqlite3_result_blob(context, p->hdr,
+ 4+8*p->nVertex, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Compute the area enclosed by the polygon.
+**
+** This routine can also be used to detect polygons that rotate in
+** the wrong direction. Polygons are suppose to be counter-clockwise (CCW).
+** This routine returns a negative value for clockwise (CW) polygons.
+*/
+static double geopolyArea(GeoPoly *p){
+ double rArea = 0.0;
+ int ii;
+ for(ii=0; iinVertex-1; ii++){
+ rArea += (p->a[ii*2] - p->a[ii*2+2]) /* (x0 - x1) */
+ * (p->a[ii*2+1] + p->a[ii*2+3]) /* (y0 + y1) */
+ * 0.5;
+ }
+ rArea += (p->a[ii*2] - p->a[0]) /* (xN - x0) */
+ * (p->a[ii*2+1] + p->a[1]) /* (yN + y0) */
+ * 0.5;
+ return rArea;
+}
+
+/*
+** Implementation of the geopoly_area(X) function.
+**
+** If the input is a well-formed Geopoly BLOB then return the area
+** enclosed by the polygon. If the polygon circulates clockwise instead
+** of counterclockwise (as it should) then return the negative of the
+** enclosed area. Otherwise return NULL.
+*/
+static void geopolyAreaFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ if( p ){
+ sqlite3_result_double(context, geopolyArea(p));
+ sqlite3_free(p);
+ }
+}
+
+/*
+** Implementation of the geopoly_ccw(X) function.
+**
+** If the rotation of polygon X is clockwise (incorrect) instead of
+** counter-clockwise (the correct winding order according to RFC7946)
+** then reverse the order of the vertexes in polygon X.
+**
+** In other words, this routine returns a CCW polygon regardless of the
+** winding order of its input.
+**
+** Use this routine to sanitize historical inputs that that sometimes
+** contain polygons that wind in the wrong direction.
+*/
+static void geopolyCcwFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyFuncParam(context, argv[0], 0);
+ if( p ){
+ if( geopolyArea(p)<0.0 ){
+ int ii, jj;
+ for(ii=2, jj=p->nVertex*2 - 2; iia[ii];
+ p->a[ii] = p->a[jj];
+ p->a[jj] = t;
+ t = p->a[ii+1];
+ p->a[ii+1] = p->a[jj+1];
+ p->a[jj+1] = t;
+ }
+ }
+ sqlite3_result_blob(context, p->hdr,
+ 4+8*p->nVertex, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+ }
+}
+
+#define GEOPOLY_PI 3.1415926535897932385
+
+/* Fast approximation for sine(X) for X between -0.5*pi and 2*pi
+*/
+static double geopolySine(double r){
+ assert( r>=-0.5*GEOPOLY_PI && r<=2.0*GEOPOLY_PI );
+ if( r>=1.5*GEOPOLY_PI ){
+ r -= 2.0*GEOPOLY_PI;
+ }
+ if( r>=0.5*GEOPOLY_PI ){
+ return -geopolySine(r-GEOPOLY_PI);
+ }else{
+ double r2 = r*r;
+ double r3 = r2*r;
+ double r5 = r3*r2;
+ return 0.9996949*r - 0.1656700*r3 + 0.0075134*r5;
+ }
+}
+
+/*
+** Function: geopoly_regular(X,Y,R,N)
+**
+** Construct a simple, convex, regular polygon centered at X, Y
+** with circumradius R and with N sides.
+*/
+static void geopolyRegularFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ double x = sqlite3_value_double(argv[0]);
+ double y = sqlite3_value_double(argv[1]);
+ double r = sqlite3_value_double(argv[2]);
+ int n = sqlite3_value_int(argv[3]);
+ int i;
+ GeoPoly *p;
+
+ if( n<3 || r<=0.0 ) return;
+ if( n>1000 ) n = 1000;
+ p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) );
+ if( p==0 ){
+ sqlite3_result_error_nomem(context);
+ return;
+ }
+ i = 1;
+ p->hdr[0] = *(unsigned char*)&i;
+ p->hdr[1] = 0;
+ p->hdr[2] = (n>>8)&0xff;
+ p->hdr[3] = n&0xff;
+ for(i=0; ia[i*2] = x - r*geopolySine(rAngle-0.5*GEOPOLY_PI);
+ p->a[i*2+1] = y + r*geopolySine(rAngle);
+ }
+ sqlite3_result_blob(context, p->hdr, 4+8*n, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+}
+
+/*
+** If pPoly is a polygon, compute its bounding box. Then:
+**
+** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL
+** (2) otherwise, compute a GeoPoly for the bounding box and return the
+** new GeoPoly
+**
+** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from
+** the bounding box in aCoord and return a pointer to that GeoPoly.
+*/
+static GeoPoly *geopolyBBox(
+ sqlite3_context *context, /* For recording the error */
+ sqlite3_value *pPoly, /* The polygon */
+ RtreeCoord *aCoord, /* Results here */
+ int *pRc /* Error code here */
+){
+ GeoPoly *pOut = 0;
+ GeoPoly *p;
+ float mnX, mxX, mnY, mxY;
+ if( pPoly==0 && aCoord!=0 ){
+ p = 0;
+ mnX = aCoord[0].f;
+ mxX = aCoord[1].f;
+ mnY = aCoord[2].f;
+ mxY = aCoord[3].f;
+ goto geopolyBboxFill;
+ }else{
+ p = geopolyFuncParam(context, pPoly, pRc);
+ }
+ if( p ){
+ int ii;
+ mnX = mxX = p->a[0];
+ mnY = mxY = p->a[1];
+ for(ii=1; iinVertex; ii++){
+ double r = p->a[ii*2];
+ if( rmxX ) mxX = (float)r;
+ r = p->a[ii*2+1];
+ if( rmxY ) mxY = (float)r;
+ }
+ if( pRc ) *pRc = SQLITE_OK;
+ if( aCoord==0 ){
+ geopolyBboxFill:
+ pOut = sqlite3_realloc(p, GEOPOLY_SZ(4));
+ if( pOut==0 ){
+ sqlite3_free(p);
+ if( context ) sqlite3_result_error_nomem(context);
+ if( pRc ) *pRc = SQLITE_NOMEM;
+ return 0;
+ }
+ pOut->nVertex = 4;
+ ii = 1;
+ pOut->hdr[0] = *(unsigned char*)ⅈ
+ pOut->hdr[1] = 0;
+ pOut->hdr[2] = 0;
+ pOut->hdr[3] = 4;
+ pOut->a[0] = mnX;
+ pOut->a[1] = mnY;
+ pOut->a[2] = mxX;
+ pOut->a[3] = mnY;
+ pOut->a[4] = mxX;
+ pOut->a[5] = mxY;
+ pOut->a[6] = mnX;
+ pOut->a[7] = mxY;
+ }else{
+ sqlite3_free(p);
+ aCoord[0].f = mnX;
+ aCoord[1].f = mxX;
+ aCoord[2].f = mnY;
+ aCoord[3].f = mxY;
+ }
+ }
+ return pOut;
+}
+
+/*
+** Implementation of the geopoly_bbox(X) SQL function.
+*/
+static void geopolyBBoxFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p = geopolyBBox(context, argv[0], 0, 0);
+ if( p ){
+ sqlite3_result_blob(context, p->hdr,
+ 4+8*p->nVertex, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+ }
+}
+
+/*
+** State vector for the geopoly_group_bbox() aggregate function.
+*/
+typedef struct GeoBBox GeoBBox;
+struct GeoBBox {
+ int isInit;
+ RtreeCoord a[4];
+};
+
+
+/*
+** Implementation of the geopoly_group_bbox(X) aggregate SQL function.
+*/
+static void geopolyBBoxStep(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ RtreeCoord a[4];
+ int rc = SQLITE_OK;
+ (void)geopolyBBox(context, argv[0], a, &rc);
+ if( rc==SQLITE_OK ){
+ GeoBBox *pBBox;
+ pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox));
+ if( pBBox==0 ) return;
+ if( pBBox->isInit==0 ){
+ pBBox->isInit = 1;
+ memcpy(pBBox->a, a, sizeof(RtreeCoord)*4);
+ }else{
+ if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0];
+ if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1];
+ if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2];
+ if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3];
+ }
+ }
+}
+static void geopolyBBoxFinal(
+ sqlite3_context *context
+){
+ GeoPoly *p;
+ GeoBBox *pBBox;
+ pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0);
+ if( pBBox==0 ) return;
+ p = geopolyBBox(context, 0, pBBox->a, 0);
+ if( p ){
+ sqlite3_result_blob(context, p->hdr,
+ 4+8*p->nVertex, SQLITE_TRANSIENT);
+ sqlite3_free(p);
+ }
+}
+
+
+/*
+** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2).
+** Returns:
+**
+** +2 x0,y0 is on the line segement
+**
+** +1 x0,y0 is beneath line segment
+**
+** 0 x0,y0 is not on or beneath the line segment or the line segment
+** is vertical and x0,y0 is not on the line segment
+**
+** The left-most coordinate min(x1,x2) is not considered to be part of
+** the line segment for the purposes of this analysis.
+*/
+static int pointBeneathLine(
+ double x0, double y0,
+ double x1, double y1,
+ double x2, double y2
+){
+ double y;
+ if( x0==x1 && y0==y1 ) return 2;
+ if( x1x2 ) return 0;
+ }else if( x1>x2 ){
+ if( x0<=x2 || x0>x1 ) return 0;
+ }else{
+ /* Vertical line segment */
+ if( x0!=x1 ) return 0;
+ if( y0y1 && y0>y2 ) return 0;
+ return 2;
+ }
+ y = y1 + (y2-y1)*(x0-x1)/(x2-x1);
+ if( y0==y ) return 2;
+ if( y0nVertex-1; ii++){
+ v = pointBeneathLine(x0,y0,p1->a[ii*2],p1->a[ii*2+1],
+ p1->a[ii*2+2],p1->a[ii*2+3]);
+ if( v==2 ) break;
+ cnt += v;
+ }
+ if( v!=2 ){
+ v = pointBeneathLine(x0,y0,p1->a[ii*2],p1->a[ii*2+1],
+ p1->a[0],p1->a[1]);
+ }
+ if( v==2 ){
+ sqlite3_result_int(context, 1);
+ }else if( ((v+cnt)&1)==0 ){
+ sqlite3_result_int(context, 0);
+ }else{
+ sqlite3_result_int(context, 2);
+ }
+ sqlite3_free(p1);
+}
+
+/* Forward declaration */
+static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2);
+
+/*
+** SQL function: geopoly_within(P1,P2)
+**
+** Return +2 if P1 and P2 are the same polygon
+** Return +1 if P2 is contained within P1
+** Return 0 if any part of P2 is on the outside of P1
+**
+*/
+static void geopolyWithinFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0);
+ GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0);
+ if( p1 && p2 ){
+ int x = geopolyOverlap(p1, p2);
+ if( x<0 ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0);
+ }
+ }
+ sqlite3_free(p1);
+ sqlite3_free(p2);
+}
+
+/* Objects used by the overlap algorihm. */
+typedef struct GeoEvent GeoEvent;
+typedef struct GeoSegment GeoSegment;
+typedef struct GeoOverlap GeoOverlap;
+struct GeoEvent {
+ double x; /* X coordinate at which event occurs */
+ int eType; /* 0 for ADD, 1 for REMOVE */
+ GeoSegment *pSeg; /* The segment to be added or removed */
+ GeoEvent *pNext; /* Next event in the sorted list */
+};
+struct GeoSegment {
+ double C, B; /* y = C*x + B */
+ double y; /* Current y value */
+ float y0; /* Initial y value */
+ unsigned char side; /* 1 for p1, 2 for p2 */
+ unsigned int idx; /* Which segment within the side */
+ GeoSegment *pNext; /* Next segment in a list sorted by y */
+};
+struct GeoOverlap {
+ GeoEvent *aEvent; /* Array of all events */
+ GeoSegment *aSegment; /* Array of all segments */
+ int nEvent; /* Number of events */
+ int nSegment; /* Number of segments */
+};
+
+/*
+** Add a single segment and its associated events.
+*/
+static void geopolyAddOneSegment(
+ GeoOverlap *p,
+ GeoCoord x0,
+ GeoCoord y0,
+ GeoCoord x1,
+ GeoCoord y1,
+ unsigned char side,
+ unsigned int idx
+){
+ GeoSegment *pSeg;
+ GeoEvent *pEvent;
+ if( x0==x1 ) return; /* Ignore vertical segments */
+ if( x0>x1 ){
+ GeoCoord t = x0;
+ x0 = x1;
+ x1 = t;
+ t = y0;
+ y0 = y1;
+ y1 = t;
+ }
+ pSeg = p->aSegment + p->nSegment;
+ p->nSegment++;
+ pSeg->C = (y1-y0)/(x1-x0);
+ pSeg->B = y1 - x1*pSeg->C;
+ pSeg->y0 = y0;
+ pSeg->side = side;
+ pSeg->idx = idx;
+ pEvent = p->aEvent + p->nEvent;
+ p->nEvent++;
+ pEvent->x = x0;
+ pEvent->eType = 0;
+ pEvent->pSeg = pSeg;
+ pEvent = p->aEvent + p->nEvent;
+ p->nEvent++;
+ pEvent->x = x1;
+ pEvent->eType = 1;
+ pEvent->pSeg = pSeg;
+}
+
+
+
+/*
+** Insert all segments and events for polygon pPoly.
+*/
+static void geopolyAddSegments(
+ GeoOverlap *p, /* Add segments to this Overlap object */
+ GeoPoly *pPoly, /* Take all segments from this polygon */
+ unsigned char side /* The side of pPoly */
+){
+ unsigned int i;
+ GeoCoord *x;
+ for(i=0; i<(unsigned)pPoly->nVertex-1; i++){
+ x = pPoly->a + (i*2);
+ geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i);
+ }
+ x = pPoly->a + (i*2);
+ geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i);
+}
+
+/*
+** Merge two lists of sorted events by X coordinate
+*/
+static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){
+ GeoEvent head, *pLast;
+ head.pNext = 0;
+ pLast = &head;
+ while( pRight && pLeft ){
+ if( pRight->x <= pLeft->x ){
+ pLast->pNext = pRight;
+ pLast = pRight;
+ pRight = pRight->pNext;
+ }else{
+ pLast->pNext = pLeft;
+ pLast = pLeft;
+ pLeft = pLeft->pNext;
+ }
+ }
+ pLast->pNext = pRight ? pRight : pLeft;
+ return head.pNext;
+}
+
+/*
+** Sort an array of nEvent event objects into a list.
+*/
+static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){
+ int mx = 0;
+ int i, j;
+ GeoEvent *p;
+ GeoEvent *a[50];
+ for(i=0; ipNext = 0;
+ for(j=0; j=mx ) mx = j+1;
+ }
+ p = 0;
+ for(i=0; iy - pLeft->y;
+ if( r==0.0 ) r = pRight->C - pLeft->C;
+ if( r<0.0 ){
+ pLast->pNext = pRight;
+ pLast = pRight;
+ pRight = pRight->pNext;
+ }else{
+ pLast->pNext = pLeft;
+ pLast = pLeft;
+ pLeft = pLeft->pNext;
+ }
+ }
+ pLast->pNext = pRight ? pRight : pLeft;
+ return head.pNext;
+}
+
+/*
+** Sort a list of GeoSegments in order of increasing Y and in the event of
+** a tie, increasing C (slope).
+*/
+static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){
+ int mx = 0;
+ int i;
+ GeoSegment *p;
+ GeoSegment *a[50];
+ while( pList ){
+ p = pList;
+ pList = pList->pNext;
+ p->pNext = 0;
+ for(i=0; i=mx ) mx = i+1;
+ }
+ p = 0;
+ for(i=0; inVertex + p2->nVertex + 2;
+ GeoOverlap *p;
+ int nByte;
+ GeoEvent *pThisEvent;
+ double rX;
+ int rc = 0;
+ int needSort = 0;
+ GeoSegment *pActive = 0;
+ GeoSegment *pSeg;
+ unsigned char aOverlap[4];
+
+ nByte = sizeof(GeoEvent)*nVertex*2
+ + sizeof(GeoSegment)*nVertex
+ + sizeof(GeoOverlap);
+ p = sqlite3_malloc( nByte );
+ if( p==0 ) return -1;
+ p->aEvent = (GeoEvent*)&p[1];
+ p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2];
+ p->nEvent = p->nSegment = 0;
+ geopolyAddSegments(p, p1, 1);
+ geopolyAddSegments(p, p2, 2);
+ pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent);
+ rX = pThisEvent->x==0.0 ? -1.0 : 0.0;
+ memset(aOverlap, 0, sizeof(aOverlap));
+ while( pThisEvent ){
+ if( pThisEvent->x!=rX ){
+ GeoSegment *pPrev = 0;
+ int iMask = 0;
+ GEODEBUG(("Distinct X: %g\n", pThisEvent->x));
+ rX = pThisEvent->x;
+ if( needSort ){
+ GEODEBUG(("SORT\n"));
+ pActive = geopolySortSegmentsByYAndC(pActive);
+ needSort = 0;
+ }
+ for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){
+ if( pPrev ){
+ if( pPrev->y!=pSeg->y ){
+ GEODEBUG(("MASK: %d\n", iMask));
+ aOverlap[iMask] = 1;
+ }
+ }
+ iMask ^= pSeg->side;
+ pPrev = pSeg;
+ }
+ pPrev = 0;
+ for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){
+ double y = pSeg->C*rX + pSeg->B;
+ GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y));
+ pSeg->y = y;
+ if( pPrev ){
+ if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){
+ rc = 1;
+ GEODEBUG(("Crossing: %d.%d and %d.%d\n",
+ pPrev->side, pPrev->idx,
+ pSeg->side, pSeg->idx));
+ goto geopolyOverlapDone;
+ }else if( pPrev->y!=pSeg->y ){
+ GEODEBUG(("MASK: %d\n", iMask));
+ aOverlap[iMask] = 1;
+ }
+ }
+ iMask ^= pSeg->side;
+ pPrev = pSeg;
+ }
+ }
+ GEODEBUG(("%s %d.%d C=%g B=%g\n",
+ pThisEvent->eType ? "RM " : "ADD",
+ pThisEvent->pSeg->side, pThisEvent->pSeg->idx,
+ pThisEvent->pSeg->C,
+ pThisEvent->pSeg->B));
+ if( pThisEvent->eType==0 ){
+ /* Add a segment */
+ pSeg = pThisEvent->pSeg;
+ pSeg->y = pSeg->y0;
+ pSeg->pNext = pActive;
+ pActive = pSeg;
+ needSort = 1;
+ }else{
+ /* Remove a segment */
+ if( pActive==pThisEvent->pSeg ){
+ pActive = pActive->pNext;
+ }else{
+ for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){
+ if( pSeg->pNext==pThisEvent->pSeg ){
+ pSeg->pNext = pSeg->pNext->pNext;
+ break;
+ }
+ }
+ }
+ }
+ pThisEvent = pThisEvent->pNext;
+ }
+ if( aOverlap[3]==0 ){
+ rc = 0;
+ }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){
+ rc = 3;
+ }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){
+ rc = 2;
+ }else if( aOverlap[1]==0 && aOverlap[2]==0 ){
+ rc = 4;
+ }else{
+ rc = 1;
+ }
+
+geopolyOverlapDone:
+ sqlite3_free(p);
+ return rc;
+}
+
+/*
+** SQL function: geopoly_overlap(P1,P2)
+**
+** Determine whether or not P1 and P2 overlap. Return value:
+**
+** 0 The two polygons are disjoint
+** 1 They overlap
+** 2 P1 is completely contained within P2
+** 3 P2 is completely contained within P1
+** 4 P1 and P2 are the same polygon
+** NULL Either P1 or P2 or both are not valid polygons
+*/
+static void geopolyOverlapFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+ GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0);
+ GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0);
+ if( p1 && p2 ){
+ int x = geopolyOverlap(p1, p2);
+ if( x<0 ){
+ sqlite3_result_error_nomem(context);
+ }else{
+ sqlite3_result_int(context, x);
+ }
+ }
+ sqlite3_free(p1);
+ sqlite3_free(p2);
+}
+
+/*
+** Enable or disable debugging output
+*/
+static void geopolyDebugFunc(
+ sqlite3_context *context,
+ int argc,
+ sqlite3_value **argv
+){
+#ifdef GEOPOLY_ENABLE_DEBUG
+ geo_debug = sqlite3_value_int(argv[0]);
+#endif
+}
+
+/*
+** This function is the implementation of both the xConnect and xCreate
+** methods of the geopoly virtual table.
+**
+** argv[0] -> module name
+** argv[1] -> database name
+** argv[2] -> table name
+** argv[...] -> column names...
+*/
+static int geopolyInit(
+ sqlite3 *db, /* Database connection */
+ void *pAux, /* One of the RTREE_COORD_* constants */
+ int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */
+ sqlite3_vtab **ppVtab, /* OUT: New virtual table */
+ char **pzErr, /* OUT: Error message, if any */
+ int isCreate /* True for xCreate, false for xConnect */
+){
+ int rc = SQLITE_OK;
+ Rtree *pRtree;
+ int nDb; /* Length of string argv[1] */
+ int nName; /* Length of string argv[2] */
+ sqlite3_str *pSql;
+ char *zSql;
+ int ii;
+
+ sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
+
+ /* Allocate the sqlite3_vtab structure */
+ nDb = (int)strlen(argv[1]);
+ nName = (int)strlen(argv[2]);
+ pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2);
+ if( !pRtree ){
+ return SQLITE_NOMEM;
+ }
+ memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2);
+ pRtree->nBusy = 1;
+ pRtree->base.pModule = &rtreeModule;
+ pRtree->zDb = (char *)&pRtree[1];
+ pRtree->zName = &pRtree->zDb[nDb+1];
+ pRtree->eCoordType = RTREE_COORD_REAL32;
+ pRtree->nDim = 2;
+ pRtree->nDim2 = 4;
+ memcpy(pRtree->zDb, argv[1], nDb);
+ memcpy(pRtree->zName, argv[2], nName);
+
+
+ /* Create/Connect to the underlying relational database schema. If
+ ** that is successful, call sqlite3_declare_vtab() to configure
+ ** the r-tree table schema.
+ */
+ pSql = sqlite3_str_new(db);
+ sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape");
+ pRtree->nAux = 1; /* Add one for _shape */
+ pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */
+ for(ii=3; iinAux++;
+ sqlite3_str_appendf(pSql, ",%s", argv[ii]);
+ }
+ sqlite3_str_appendf(pSql, ");");
+ zSql = sqlite3_str_finish(pSql);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+ }
+ sqlite3_free(zSql);
+ if( rc ) goto geopolyInit_fail;
+ pRtree->nBytesPerCell = 8 + pRtree->nDim2*4;
+
+ /* Figure out the node size to use. */
+ rc = getNodeSize(db, pRtree, isCreate, pzErr);
+ if( rc ) goto geopolyInit_fail;
+ rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate);
+ if( rc ){
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+ goto geopolyInit_fail;
+ }
+
+ *ppVtab = (sqlite3_vtab *)pRtree;
+ return SQLITE_OK;
+
+geopolyInit_fail:
+ if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
+ assert( *ppVtab==0 );
+ assert( pRtree->nBusy==1 );
+ rtreeRelease(pRtree);
+ return rc;
+}
+
+
+/*
+** GEOPOLY virtual table module xCreate method.
+*/
+static int geopolyCreate(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1);
+}
+
+/*
+** GEOPOLY virtual table module xConnect method.
+*/
+static int geopolyConnect(
+ sqlite3 *db,
+ void *pAux,
+ int argc, const char *const*argv,
+ sqlite3_vtab **ppVtab,
+ char **pzErr
+){
+ return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0);
+}
+
+
+/*
+** GEOPOLY virtual table module xFilter method.
+**
+** Query plans:
+**
+** 1 rowid lookup
+** 2 search for objects overlapping the same bounding box
+** that contains polygon argv[0]
+** 3 search for objects overlapping the same bounding box
+** that contains polygon argv[0]
+** 4 full table scan
+*/
+static int geopolyFilter(
+ sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */
+ int idxNum, /* Query plan */
+ const char *idxStr, /* Not Used */
+ int argc, sqlite3_value **argv /* Parameters to the query plan */
+){
+ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab;
+ RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor;
+ RtreeNode *pRoot = 0;
+ int rc = SQLITE_OK;
+ int iCell = 0;
+ sqlite3_stmt *pStmt;
+
+ rtreeReference(pRtree);
+
+ /* Reset the cursor to the same state as rtreeOpen() leaves it in. */
+ freeCursorConstraints(pCsr);
+ sqlite3_free(pCsr->aPoint);
+ pStmt = pCsr->pReadAux;
+ memset(pCsr, 0, sizeof(RtreeCursor));
+ pCsr->base.pVtab = (sqlite3_vtab*)pRtree;
+ pCsr->pReadAux = pStmt;
+
+ pCsr->iStrategy = idxNum;
+ if( idxNum==1 ){
+ /* Special case - lookup by rowid. */
+ RtreeNode *pLeaf; /* Leaf on which the required cell resides */
+ RtreeSearchPoint *p; /* Search point for the leaf */
+ i64 iRowid = sqlite3_value_int64(argv[0]);
+ i64 iNode = 0;
+ rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode);
+ if( rc==SQLITE_OK && pLeaf!=0 ){
+ p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0);
+ assert( p!=0 ); /* Always returns pCsr->sPoint */
+ pCsr->aNode[0] = pLeaf;
+ p->id = iNode;
+ p->eWithin = PARTLY_WITHIN;
+ rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell);
+ p->iCell = (u8)iCell;
+ RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:");
+ }else{
+ pCsr->atEOF = 1;
+ }
+ }else{
+ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array
+ ** with the configured constraints.
+ */
+ rc = nodeAcquire(pRtree, 1, 0, &pRoot);
+ if( rc==SQLITE_OK && idxNum<=3 ){
+ RtreeCoord bbox[4];
+ RtreeConstraint *p;
+ assert( argc==1 );
+ geopolyBBox(0, argv[0], bbox, &rc);
+ if( rc ){
+ goto geopoly_filter_end;
+ }
+ pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4);
+ pCsr->nConstraint = 4;
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4);
+ memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1));
+ if( idxNum==2 ){
+ /* Overlap query */
+ p->op = 'B';
+ p->iCoord = 0;
+ p->u.rValue = bbox[1].f;
+ p++;
+ p->op = 'D';
+ p->iCoord = 1;
+ p->u.rValue = bbox[0].f;
+ p++;
+ p->op = 'B';
+ p->iCoord = 2;
+ p->u.rValue = bbox[3].f;
+ p++;
+ p->op = 'D';
+ p->iCoord = 3;
+ p->u.rValue = bbox[2].f;
+ }else{
+ /* Within query */
+ p->op = 'D';
+ p->iCoord = 0;
+ p->u.rValue = bbox[0].f;
+ p++;
+ p->op = 'B';
+ p->iCoord = 1;
+ p->u.rValue = bbox[1].f;
+ p++;
+ p->op = 'D';
+ p->iCoord = 2;
+ p->u.rValue = bbox[2].f;
+ p++;
+ p->op = 'B';
+ p->iCoord = 3;
+ p->u.rValue = bbox[3].f;
+ }
+ }
+ }
+ if( rc==SQLITE_OK ){
+ RtreeSearchPoint *pNew;
+ pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1));
+ if( pNew==0 ){
+ rc = SQLITE_NOMEM;
+ goto geopoly_filter_end;
+ }
+ pNew->id = 1;
+ pNew->iCell = 0;
+ pNew->eWithin = PARTLY_WITHIN;
+ assert( pCsr->bPoint==1 );
+ pCsr->aNode[0] = pRoot;
+ pRoot = 0;
+ RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:");
+ rc = rtreeStepToLeaf(pCsr);
+ }
+ }
+
+geopoly_filter_end:
+ nodeRelease(pRtree, pRoot);
+ rtreeRelease(pRtree);
+ return rc;
+}
+
+/*
+** Rtree virtual table module xBestIndex method. There are three
+** table scan strategies to choose from (in order from most to
+** least desirable):
+**
+** idxNum idxStr Strategy
+** ------------------------------------------------
+** 1 "rowid" Direct lookup by rowid.
+** 2 "rtree" R-tree overlap query using geopoly_overlap()
+** 3 "rtree" R-tree within query using geopoly_within()
+** 4 "fullscan" full-table scan.
+** ------------------------------------------------
+*/
+static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
+ int ii;
+ int iRowidTerm = -1;
+ int iFuncTerm = -1;
+ int idxNum = 0;
+
+ for(ii=0; iinConstraint; ii++){
+ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii];
+ if( !p->usable ) continue;
+ if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+ iRowidTerm = ii;
+ break;
+ }
+ if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){
+ /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap()
+ ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within().
+ ** See geopolyFindFunction() */
+ iFuncTerm = ii;
+ idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2;
+ }
+ }
+
+ if( iRowidTerm>=0 ){
+ pIdxInfo->idxNum = 1;
+ pIdxInfo->idxStr = "rowid";
+ pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1;
+ pIdxInfo->estimatedCost = 30.0;
+ pIdxInfo->estimatedRows = 1;
+ pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE;
+ return SQLITE_OK;
+ }
+ if( iFuncTerm>=0 ){
+ pIdxInfo->idxNum = idxNum;
+ pIdxInfo->idxStr = "rtree";
+ pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1;
+ pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0;
+ pIdxInfo->estimatedCost = 300.0;
+ pIdxInfo->estimatedRows = 10;
+ return SQLITE_OK;
+ }
+ pIdxInfo->idxNum = 4;
+ pIdxInfo->idxStr = "fullscan";
+ pIdxInfo->estimatedCost = 3000000.0;
+ pIdxInfo->estimatedRows = 100000;
+ return SQLITE_OK;
+}
+
+
+/*
+** GEOPOLY virtual table module xColumn method.
+*/
+static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
+ Rtree *pRtree = (Rtree *)cur->pVtab;
+ RtreeCursor *pCsr = (RtreeCursor *)cur;
+ RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr);
+ int rc = SQLITE_OK;
+ RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc);
+
+ if( rc ) return rc;
+ if( p==0 ) return SQLITE_OK;
+ if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK;
+ if( i<=pRtree->nAux ){
+ if( !pCsr->bAuxValid ){
+ if( pCsr->pReadAux==0 ){
+ rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0,
+ &pCsr->pReadAux, 0);
+ if( rc ) return rc;
+ }
+ sqlite3_bind_int64(pCsr->pReadAux, 1,
+ nodeGetRowid(pRtree, pNode, p->iCell));
+ rc = sqlite3_step(pCsr->pReadAux);
+ if( rc==SQLITE_ROW ){
+ pCsr->bAuxValid = 1;
+ }else{
+ sqlite3_reset(pCsr->pReadAux);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ return rc;
+ }
+ }
+ sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2));
+ }
+ return SQLITE_OK;
+}
+
+
+/*
+** The xUpdate method for GEOPOLY module virtual tables.
+**
+** For DELETE:
+**
+** argv[0] = the rowid to be deleted
+**
+** For INSERT:
+**
+** argv[0] = SQL NULL
+** argv[1] = rowid to insert, or an SQL NULL to select automatically
+** argv[2] = _shape column
+** argv[3] = first application-defined column....
+**
+** For UPDATE:
+**
+** argv[0] = rowid to modify. Never NULL
+** argv[1] = rowid after the change. Never NULL
+** argv[2] = new value for _shape
+** argv[3] = new value for first application-defined column....
+*/
+static int geopolyUpdate(
+ sqlite3_vtab *pVtab,
+ int nData,
+ sqlite3_value **aData,
+ sqlite_int64 *pRowid
+){
+ Rtree *pRtree = (Rtree *)pVtab;
+ int rc = SQLITE_OK;
+ RtreeCell cell; /* New cell to insert if nData>1 */
+ i64 oldRowid; /* The old rowid */
+ int oldRowidValid; /* True if oldRowid is valid */
+ i64 newRowid; /* The new rowid */
+ int newRowidValid; /* True if newRowid is valid */
+ int coordChange = 0; /* Change in coordinates */
+
+ if( pRtree->nNodeRef ){
+ /* Unable to write to the btree while another cursor is reading from it,
+ ** since the write might do a rebalance which would disrupt the read
+ ** cursor. */
+ return SQLITE_LOCKED_VTAB;
+ }
+ rtreeReference(pRtree);
+ assert(nData>=1);
+
+ oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;;
+ oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0;
+ newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL;
+ newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0;
+ cell.iRowid = newRowid;
+
+ if( nData>1 /* not a DELETE */
+ && (!oldRowidValid /* INSERT */
+ || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */
+ || oldRowid!=newRowid) /* Rowid change */
+ ){
+ geopolyBBox(0, aData[2], cell.aCoord, &rc);
+ if( rc ){
+ if( rc==SQLITE_ERROR ){
+ pVtab->zErrMsg =
+ sqlite3_mprintf("_shape does not contain a valid polygon");
+ }
+ goto geopoly_update_end;
+ }
+ coordChange = 1;
+
+ /* If a rowid value was supplied, check if it is already present in
+ ** the table. If so, the constraint has failed. */
+ if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){
+ int steprc;
+ sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid);
+ steprc = sqlite3_step(pRtree->pReadRowid);
+ rc = sqlite3_reset(pRtree->pReadRowid);
+ if( SQLITE_ROW==steprc ){
+ if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){
+ rc = rtreeDeleteRowid(pRtree, cell.iRowid);
+ }else{
+ rc = rtreeConstraintError(pRtree, 0);
+ }
+ }
+ }
+ }
+
+ /* If aData[0] is not an SQL NULL value, it is the rowid of a
+ ** record to delete from the r-tree table. The following block does
+ ** just that.
+ */
+ if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){
+ rc = rtreeDeleteRowid(pRtree, oldRowid);
+ }
+
+ /* If the aData[] array contains more than one element, elements
+ ** (aData[2]..aData[argc-1]) contain a new record to insert into
+ ** the r-tree structure.
+ */
+ if( rc==SQLITE_OK && nData>1 && coordChange ){
+ /* Insert the new record into the r-tree */
+ RtreeNode *pLeaf = 0;
+ if( !newRowidValid ){
+ rc = rtreeNewRowid(pRtree, &cell.iRowid);
+ }
+ *pRowid = cell.iRowid;
+ if( rc==SQLITE_OK ){
+ rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
+ }
+ if( rc==SQLITE_OK ){
+ int rc2;
+ pRtree->iReinsertHeight = -1;
+ rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0);
+ rc2 = nodeRelease(pRtree, pLeaf);
+ if( rc==SQLITE_OK ){
+ rc = rc2;
+ }
+ }
+ }
+
+ /* Change the data */
+ if( rc==SQLITE_OK && nData>1 ){
+ sqlite3_stmt *pUp = pRtree->pWriteAux;
+ int jj;
+ int nChange = 0;
+ sqlite3_bind_int64(pUp, 1, cell.iRowid);
+ assert( pRtree->nAux>=1 );
+ if( sqlite3_value_nochange(aData[2]) ){
+ sqlite3_bind_null(pUp, 2);
+ }else{
+ GeoPoly *p = 0;
+ if( sqlite3_value_type(aData[2])==SQLITE_TEXT
+ && (p = geopolyFuncParam(0, aData[2], &rc))!=0
+ && rc==SQLITE_OK
+ ){
+ sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT);
+ }else{
+ sqlite3_bind_value(pUp, 2, aData[2]);
+ }
+ sqlite3_free(p);
+ nChange = 1;
+ }
+ for(jj=1; jjnAux; jj++){
+ nChange++;
+ sqlite3_bind_value(pUp, jj+2, aData[jj+2]);
+ }
+ if( nChange ){
+ sqlite3_step(pUp);
+ rc = sqlite3_reset(pUp);
+ }
+ }
+
+geopoly_update_end:
+ rtreeRelease(pRtree);
+ return rc;
+}
+
+/*
+** Report that geopoly_overlap() is an overloaded function suitable
+** for use in xBestIndex.
+*/
+static int geopolyFindFunction(
+ sqlite3_vtab *pVtab,
+ int nArg,
+ const char *zName,
+ void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
+ void **ppArg
+){
+ if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){
+ *pxFunc = geopolyOverlapFunc;
+ *ppArg = 0;
+ return SQLITE_INDEX_CONSTRAINT_FUNCTION;
+ }
+ if( sqlite3_stricmp(zName, "geopoly_within")==0 ){
+ *pxFunc = geopolyWithinFunc;
+ *ppArg = 0;
+ return SQLITE_INDEX_CONSTRAINT_FUNCTION+1;
+ }
+ return 0;
+}
+
+
+static sqlite3_module geopolyModule = {
+ 3, /* iVersion */
+ geopolyCreate, /* xCreate - create a table */
+ geopolyConnect, /* xConnect - connect to an existing table */
+ geopolyBestIndex, /* xBestIndex - Determine search strategy */
+ rtreeDisconnect, /* xDisconnect - Disconnect from a table */
+ rtreeDestroy, /* xDestroy - Drop a table */
+ rtreeOpen, /* xOpen - open a cursor */
+ rtreeClose, /* xClose - close a cursor */
+ geopolyFilter, /* xFilter - configure scan constraints */
+ rtreeNext, /* xNext - advance a cursor */
+ rtreeEof, /* xEof */
+ geopolyColumn, /* xColumn - read data */
+ rtreeRowid, /* xRowid - read data */
+ geopolyUpdate, /* xUpdate - write data */
+ rtreeBeginTransaction, /* xBegin - begin transaction */
+ rtreeEndTransaction, /* xSync - sync transaction */
+ rtreeEndTransaction, /* xCommit - commit transaction */
+ rtreeEndTransaction, /* xRollback - rollback transaction */
+ geopolyFindFunction, /* xFindFunction - function overloading */
+ rtreeRename, /* xRename - rename the table */
+ rtreeSavepoint, /* xSavepoint */
+ 0, /* xRelease */
+ 0, /* xRollbackTo */
+ rtreeShadowName /* xShadowName */
+};
+
+static int sqlite3_geopoly_init(sqlite3 *db){
+ int rc = SQLITE_OK;
+ static const struct {
+ void (*xFunc)(sqlite3_context*,int,sqlite3_value**);
+ signed char nArg;
+ unsigned char bPure;
+ const char *zName;
+ } aFunc[] = {
+ { geopolyAreaFunc, 1, 1, "geopoly_area" },
+ { geopolyBlobFunc, 1, 1, "geopoly_blob" },
+ { geopolyJsonFunc, 1, 1, "geopoly_json" },
+ { geopolySvgFunc, -1, 1, "geopoly_svg" },
+ { geopolyWithinFunc, 2, 1, "geopoly_within" },
+ { geopolyContainsPointFunc, 3, 1, "geopoly_contains_point" },
+ { geopolyOverlapFunc, 2, 1, "geopoly_overlap" },
+ { geopolyDebugFunc, 1, 0, "geopoly_debug" },
+ { geopolyBBoxFunc, 1, 1, "geopoly_bbox" },
+ { geopolyXformFunc, 7, 1, "geopoly_xform" },
+ { geopolyRegularFunc, 4, 1, "geopoly_regular" },
+ { geopolyCcwFunc, 1, 1, "geopoly_ccw" },
+ };
+ static const struct {
+ void (*xStep)(sqlite3_context*,int,sqlite3_value**);
+ void (*xFinal)(sqlite3_context*);
+ const char *zName;
+ } aAgg[] = {
+ { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" },
+ };
+ int i;
+ for(i=0; inRef>0 );
p->nRef++;
}
}
/*
@@ -587,10 +602,11 @@
pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize);
if( pNode ){
memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize);
pNode->zData = (u8 *)&pNode[1];
pNode->nRef = 1;
+ pRtree->nNodeRef++;
pNode->pParent = pParent;
pNode->isDirty = 1;
nodeReference(pParent);
}
return pNode;
@@ -620,14 +636,14 @@
RtreeNode *pNode = 0;
/* Check if the requested node is already in the hash table. If so,
** increase its reference count and return it.
*/
- if( (pNode = nodeHashLookup(pRtree, iNode)) ){
+ if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){
assert( !pParent || !pNode->pParent || pNode->pParent==pParent );
if( pParent && !pNode->pParent ){
- nodeReference(pParent);
+ pParent->nRef++;
pNode->pParent = pParent;
}
pNode->nRef++;
*ppNode = pNode;
return SQLITE_OK;
@@ -662,10 +678,11 @@
rc = SQLITE_NOMEM;
}else{
pNode->pParent = pParent;
pNode->zData = (u8 *)&pNode[1];
pNode->nRef = 1;
+ pRtree->nNodeRef++;
pNode->iNode = iNode;
pNode->isDirty = 0;
pNode->pNext = 0;
rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData,
pRtree->iNodeSize, 0);
@@ -702,11 +719,14 @@
}else{
rc = SQLITE_CORRUPT_VTAB;
}
*ppNode = pNode;
}else{
- sqlite3_free(pNode);
+ if( pNode ){
+ pRtree->nNodeRef--;
+ sqlite3_free(pNode);
+ }
*ppNode = 0;
}
return rc;
}
@@ -782,10 +802,11 @@
}
sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC);
sqlite3_step(p);
pNode->isDirty = 0;
rc = sqlite3_reset(p);
+ sqlite3_bind_null(p, 2);
if( pNode->iNode==0 && rc==SQLITE_OK ){
pNode->iNode = sqlite3_last_insert_rowid(pRtree->db);
nodeHashInsert(pRtree, pNode);
}
}
@@ -798,12 +819,14 @@
*/
static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){
int rc = SQLITE_OK;
if( pNode ){
assert( pNode->nRef>0 );
+ assert( pRtree->nNodeRef>0 );
pNode->nRef--;
if( pNode->nRef==0 ){
+ pRtree->nNodeRef--;
if( pNode->iNode==1 ){
pRtree->iDepth = -1;
}
if( pNode->pParent ){
rc = nodeRelease(pRtree, pNode->pParent);
@@ -916,20 +939,23 @@
*/
static void rtreeRelease(Rtree *pRtree){
pRtree->nBusy--;
if( pRtree->nBusy==0 ){
pRtree->inWrTrans = 0;
- pRtree->nCursor = 0;
+ assert( pRtree->nCursor==0 );
nodeBlobReset(pRtree);
+ assert( pRtree->nNodeRef==0 );
sqlite3_finalize(pRtree->pWriteNode);
sqlite3_finalize(pRtree->pDeleteNode);
sqlite3_finalize(pRtree->pReadRowid);
sqlite3_finalize(pRtree->pWriteRowid);
sqlite3_finalize(pRtree->pDeleteRowid);
sqlite3_finalize(pRtree->pReadParent);
sqlite3_finalize(pRtree->pWriteParent);
sqlite3_finalize(pRtree->pDeleteParent);
+ sqlite3_finalize(pRtree->pWriteAux);
+ sqlite3_free(pRtree->zReadAuxSql);
sqlite3_free(pRtree);
}
}
/*
@@ -1014,10 +1040,11 @@
Rtree *pRtree = (Rtree *)(cur->pVtab);
int ii;
RtreeCursor *pCsr = (RtreeCursor *)cur;
assert( pRtree->nCursor>0 );
freeCursorConstraints(pCsr);
+ sqlite3_finalize(pCsr->pReadAux);
sqlite3_free(pCsr->aPoint);
for(ii=0; iiaNode[ii]);
sqlite3_free(pCsr);
pRtree->nCursor--;
nodeBlobReset(pRtree);
@@ -1385,11 +1412,11 @@
if( pNew==0 ) return 0;
ii = (int)(pNew - pCur->aPoint) + 1;
if( iiaNode[ii]==0 );
pCur->aNode[ii] = pCur->aNode[0];
- }else{
+ }else{
nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]);
}
pCur->aNode[0] = 0;
*pNew = pCur->sPoint;
}
@@ -1556,10 +1583,14 @@
RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor;
int rc = SQLITE_OK;
/* Move to the next entry that matches the configured constraints. */
RTREE_QUEUE_TRACE(pCsr, "POP-Nx:");
+ if( pCsr->bAuxValid ){
+ pCsr->bAuxValid = 0;
+ sqlite3_reset(pCsr->pReadAux);
+ }
rtreeSearchPointPop(pCsr);
rc = rtreeStepToLeaf(pCsr);
return rc;
}
@@ -1590,11 +1621,11 @@
if( rc ) return rc;
if( p==0 ) return SQLITE_OK;
if( i==0 ){
sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell));
- }else{
+ }else if( i<=pRtree->nDim2 ){
nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c);
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
sqlite3_result_double(ctx, c.f);
}else
@@ -1601,11 +1632,31 @@
#endif
{
assert( pRtree->eCoordType==RTREE_COORD_INT32 );
sqlite3_result_int(ctx, c.i);
}
- }
+ }else{
+ if( !pCsr->bAuxValid ){
+ if( pCsr->pReadAux==0 ){
+ rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0,
+ &pCsr->pReadAux, 0);
+ if( rc ) return rc;
+ }
+ sqlite3_bind_int64(pCsr->pReadAux, 1,
+ nodeGetRowid(pRtree, pNode, p->iCell));
+ rc = sqlite3_step(pCsr->pReadAux);
+ if( rc==SQLITE_ROW ){
+ pCsr->bAuxValid = 1;
+ }else{
+ sqlite3_reset(pCsr->pReadAux);
+ if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+ return rc;
+ }
+ }
+ sqlite3_result_value(ctx,
+ sqlite3_column_value(pCsr->pReadAux, i - pRtree->nDim2 + 1));
+ }
return SQLITE_OK;
}
/*
** Use nodeAcquire() to obtain the leaf node containing the record with
@@ -1679,18 +1730,21 @@
RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor;
RtreeNode *pRoot = 0;
int ii;
int rc = SQLITE_OK;
int iCell = 0;
+ sqlite3_stmt *pStmt;
rtreeReference(pRtree);
/* Reset the cursor to the same state as rtreeOpen() leaves it in. */
freeCursorConstraints(pCsr);
sqlite3_free(pCsr->aPoint);
+ pStmt = pCsr->pReadAux;
memset(pCsr, 0, sizeof(RtreeCursor));
pCsr->base.pVtab = (sqlite3_vtab*)pRtree;
+ pCsr->pReadAux = pStmt;
pCsr->iStrategy = idxNum;
if( idxNum==1 ){
/* Special case - lookup by rowid. */
RtreeNode *pLeaf; /* Leaf on which the required cell resides */
@@ -1849,14 +1903,18 @@
** sqlite uses an internal cost of 0.0). It is expected to return
** a single row.
*/
pIdxInfo->estimatedCost = 30.0;
pIdxInfo->estimatedRows = 1;
+ pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE;
return SQLITE_OK;
}
- if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){
+ if( p->usable
+ && ((p->iColumn>0 && p->iColumn<=pRtree->nDim2)
+ || p->op==SQLITE_INDEX_CONSTRAINT_MATCH)
+ ){
u8 op;
switch( p->op ){
case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break;
case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break;
case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break;
@@ -2019,11 +2077,11 @@
int iHeight, /* Height of sub-tree rooted at pCell */
RtreeNode **ppLeaf /* OUT: Selected leaf page */
){
int rc;
int ii;
- RtreeNode *pNode;
+ RtreeNode *pNode = 0;
rc = nodeAcquire(pRtree, 1, 0, &pNode);
for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){
int iCell;
sqlite3_int64 iBest = 0;
@@ -2425,11 +2483,11 @@
pNode->isDirty = 1;
writeInt16(pNode->zData, pRtree->iDepth);
}else{
pLeft = pNode;
pRight = nodeNew(pRtree, pLeft->pParent);
- nodeReference(pLeft);
+ pLeft->nRef++;
}
if( !pLeft || !pRight ){
rc = SQLITE_NOMEM;
goto splitnode_out;
@@ -2834,11 +2892,11 @@
}
/*
** Select a currently unused rowid for a new r-tree record.
*/
-static int newRowid(Rtree *pRtree, i64 *piRowid){
+static int rtreeNewRowid(Rtree *pRtree, i64 *piRowid){
int rc;
sqlite3_bind_null(pRtree->pWriteRowid, 1);
sqlite3_bind_null(pRtree->pWriteRowid, 2);
sqlite3_step(pRtree->pWriteRowid);
rc = sqlite3_reset(pRtree->pWriteRowid);
@@ -2894,11 +2952,11 @@
** the root node (the operation that Gutman's paper says to perform
** in this scenario).
*/
if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){
int rc2;
- RtreeNode *pChild;
+ RtreeNode *pChild = 0;
i64 iChild = nodeGetRowid(pRtree, pRoot, 0);
rc = nodeAcquire(pRtree, iChild, pRoot, &pChild);
if( rc==SQLITE_OK ){
rc = removeNode(pRtree, pChild, pRtree->iDepth-1);
}
@@ -2915,10 +2973,11 @@
for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){
if( rc==SQLITE_OK ){
rc = reinsertNodeContent(pRtree, pLeaf);
}
pRtree->pDeleted = pLeaf->pNext;
+ pRtree->nNodeRef--;
sqlite3_free(pLeaf);
}
/* Release the reference to the root node. */
if( rc==SQLITE_OK ){
@@ -3011,18 +3070,24 @@
** The xUpdate method for rtree module virtual tables.
*/
static int rtreeUpdate(
sqlite3_vtab *pVtab,
int nData,
- sqlite3_value **azData,
+ sqlite3_value **aData,
sqlite_int64 *pRowid
){
Rtree *pRtree = (Rtree *)pVtab;
int rc = SQLITE_OK;
RtreeCell cell; /* New cell to insert if nData>1 */
int bHaveRowid = 0; /* Set to 1 after new rowid is determined */
+ if( pRtree->nNodeRef ){
+ /* Unable to write to the btree while another cursor is reading from it,
+ ** since the write might do a rebalance which would disrupt the read
+ ** cursor. */
+ return SQLITE_LOCKED_VTAB;
+ }
rtreeReference(pRtree);
assert(nData>=1);
cell.iRowid = 0; /* Used only to suppress a compiler warning */
@@ -3037,50 +3102,51 @@
** case, SQLITE_CONSTRAINT must be returned regardless of the
** conflict-handling mode specified by the user.
*/
if( nData>1 ){
int ii;
+ int nn = nData - 4;
- /* Populate the cell.aCoord[] array. The first coordinate is azData[3].
+ if( nn > pRtree->nDim2 ) nn = pRtree->nDim2;
+ /* Populate the cell.aCoord[] array. The first coordinate is aData[3].
**
** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared
** with "column" that are interpreted as table constraints.
** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5));
** This problem was discovered after years of use, so we silently ignore
** these kinds of misdeclared tables to avoid breaking any legacy.
*/
- assert( nData<=(pRtree->nDim2 + 3) );
#ifndef SQLITE_RTREE_INT_ONLY
if( pRtree->eCoordType==RTREE_COORD_REAL32 ){
- for(ii=0; iicell.aCoord[ii+1].f ){
rc = rtreeConstraintError(pRtree, ii+1);
goto constraint;
}
}
}else
#endif
{
- for(ii=0; iicell.aCoord[ii+1].i ){
rc = rtreeConstraintError(pRtree, ii+1);
goto constraint;
}
}
}
/* If a rowid value was supplied, check if it is already present in
** the table. If so, the constraint has failed. */
- if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){
- cell.iRowid = sqlite3_value_int64(azData[2]);
- if( sqlite3_value_type(azData[0])==SQLITE_NULL
- || sqlite3_value_int64(azData[0])!=cell.iRowid
+ if( sqlite3_value_type(aData[2])!=SQLITE_NULL ){
+ cell.iRowid = sqlite3_value_int64(aData[2]);
+ if( sqlite3_value_type(aData[0])==SQLITE_NULL
+ || sqlite3_value_int64(aData[0])!=cell.iRowid
){
int steprc;
sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid);
steprc = sqlite3_step(pRtree->pReadRowid);
rc = sqlite3_reset(pRtree->pReadRowid);
@@ -3095,29 +3161,29 @@
}
bHaveRowid = 1;
}
}
- /* If azData[0] is not an SQL NULL value, it is the rowid of a
+ /* If aData[0] is not an SQL NULL value, it is the rowid of a
** record to delete from the r-tree table. The following block does
** just that.
*/
- if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){
- rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0]));
+ if( sqlite3_value_type(aData[0])!=SQLITE_NULL ){
+ rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(aData[0]));
}
- /* If the azData[] array contains more than one element, elements
- ** (azData[2]..azData[argc-1]) contain a new record to insert into
+ /* If the aData[] array contains more than one element, elements
+ ** (aData[2]..aData[argc-1]) contain a new record to insert into
** the r-tree structure.
*/
if( rc==SQLITE_OK && nData>1 ){
/* Insert the new record into the r-tree */
RtreeNode *pLeaf = 0;
/* Figure out the rowid of the new row. */
if( bHaveRowid==0 ){
- rc = newRowid(pRtree, &cell.iRowid);
+ rc = rtreeNewRowid(pRtree, &cell.iRowid);
}
*pRowid = cell.iRowid;
if( rc==SQLITE_OK ){
rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf);
@@ -3129,10 +3195,20 @@
rc2 = nodeRelease(pRtree, pLeaf);
if( rc==SQLITE_OK ){
rc = rc2;
}
}
+ if( pRtree->nAux ){
+ sqlite3_stmt *pUp = pRtree->pWriteAux;
+ int jj;
+ sqlite3_bind_int64(pUp, 1, *pRowid);
+ for(jj=0; jjnAux; jj++){
+ sqlite3_bind_value(pUp, jj+2, aData[pRtree->nDim2+3+jj]);
+ }
+ sqlite3_step(pUp);
+ rc = sqlite3_reset(pUp);
+ }
}
constraint:
rtreeRelease(pRtree);
return rc;
@@ -3195,11 +3271,11 @@
** DROP TABLE ; -- Would fail with SQLITE_LOCKED
** COMMIT;
*/
static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){
Rtree *pRtree = (Rtree *)pVtab;
- int iwt = pRtree->inWrTrans;
+ u8 iwt = pRtree->inWrTrans;
UNUSED_PARAMETER(iSavepoint);
pRtree->inWrTrans = 0;
nodeBlobReset(pRtree);
pRtree->inWrTrans = iwt;
return SQLITE_OK;
@@ -3247,12 +3323,28 @@
}
return rc;
}
+
+/*
+** Return true if zName is the extension on one of the shadow tables used
+** by this module.
+*/
+static int rtreeShadowName(const char *zName){
+ static const char *azName[] = {
+ "node", "parent", "rowid"
+ };
+ unsigned int i;
+ for(i=0; idb = db;
if( isCreate ){
- char *zCreate = sqlite3_mprintf(
-"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);"
-"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);"
-"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY,"
- " parentnode INTEGER);"
-"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))",
- zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize
- );
+ char *zCreate;
+ sqlite3_str *p = sqlite3_str_new(db);
+ int ii;
+ sqlite3_str_appendf(p,
+ "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY,nodeno",
+ zDb, zPrefix);
+ for(ii=0; iinAux; ii++){
+ sqlite3_str_appendf(p,",a%d",ii);
+ }
+ sqlite3_str_appendf(p,
+ ");CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY,data);",
+ zDb, zPrefix);
+ sqlite3_str_appendf(p,
+ "CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY,parentnode);",
+ zDb, zPrefix);
+ sqlite3_str_appendf(p,
+ "INSERT INTO \"%w\".\"%w_node\"VALUES(1,zeroblob(%d))",
+ zDb, zPrefix, pRtree->iNodeSize);
+ zCreate = sqlite3_str_finish(p);
if( !zCreate ){
return SQLITE_NOMEM;
}
rc = sqlite3_exec(db, zCreate, 0, 0, 0);
sqlite3_free(zCreate);
@@ -3333,18 +3437,58 @@
appStmt[6] = &pRtree->pWriteParent;
appStmt[7] = &pRtree->pDeleteParent;
rc = rtreeQueryStat1(db, pRtree);
for(i=0; inAux==0 ){
+ zFormat = azSql[i];
+ }else {
+ /* An UPSERT is very slightly slower than REPLACE, but it is needed
+ ** if there are auxiliary columns */
+ zFormat = "INSERT INTO\"%w\".\"%w_rowid\"(rowid,nodeno)VALUES(?1,?2)"
+ "ON CONFLICT(rowid)DO UPDATE SET nodeno=excluded.nodeno";
+ }
+ zSql = sqlite3_mprintf(zFormat, zDb, zPrefix);
if( zSql ){
rc = sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
appStmt[i], 0);
}else{
rc = SQLITE_NOMEM;
}
sqlite3_free(zSql);
+ }
+ if( pRtree->nAux ){
+ pRtree->zReadAuxSql = sqlite3_mprintf(
+ "SELECT * FROM \"%w\".\"%w_rowid\" WHERE rowid=?1",
+ zDb, zPrefix);
+ if( pRtree->zReadAuxSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ sqlite3_str *p = sqlite3_str_new(db);
+ int ii;
+ char *zSql;
+ sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix);
+ for(ii=0; iinAux; ii++){
+ if( ii ) sqlite3_str_append(p, ",", 1);
+ if( iinAuxNotNull ){
+ sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii);
+ }else{
+ sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2);
+ }
+ }
+ sqlite3_str_appendf(p, " WHERE rowid=?1");
+ zSql = sqlite3_str_finish(p);
+ if( zSql==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_PERSISTENT,
+ &pRtree->pWriteAux, 0);
+ sqlite3_free(zSql);
+ }
+ }
}
return rc;
}
@@ -3444,21 +3588,26 @@
int rc = SQLITE_OK;
Rtree *pRtree;
int nDb; /* Length of string argv[1] */
int nName; /* Length of string argv[2] */
int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32);
+ sqlite3_str *pSql;
+ char *zSql;
+ int ii = 4;
+ int iErr;
const char *aErrMsg[] = {
0, /* 0 */
"Wrong number of columns for an rtree table", /* 1 */
"Too few columns for an rtree table", /* 2 */
- "Too many columns for an rtree table" /* 3 */
+ "Too many columns for an rtree table", /* 3 */
+ "Auxiliary rtree columns must be last" /* 4 */
};
- int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2;
- if( aErrMsg[iErr] ){
- *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]);
+ assert( RTREE_MAX_AUX_COLUMN<256 ); /* Aux columns counted by a u8 */
+ if( argc>RTREE_MAX_AUX_COLUMN+3 ){
+ *pzErr = sqlite3_mprintf("%s", aErrMsg[3]);
return SQLITE_ERROR;
}
sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
@@ -3472,57 +3621,77 @@
memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2);
pRtree->nBusy = 1;
pRtree->base.pModule = &rtreeModule;
pRtree->zDb = (char *)&pRtree[1];
pRtree->zName = &pRtree->zDb[nDb+1];
- pRtree->nDim = (u8)((argc-4)/2);
- pRtree->nDim2 = pRtree->nDim*2;
- pRtree->nBytesPerCell = 8 + pRtree->nDim2*4;
pRtree->eCoordType = (u8)eCoordType;
memcpy(pRtree->zDb, argv[1], nDb);
memcpy(pRtree->zName, argv[2], nName);
- /* Figure out the node size to use. */
- rc = getNodeSize(db, pRtree, isCreate, pzErr);
/* Create/Connect to the underlying relational database schema. If
** that is successful, call sqlite3_declare_vtab() to configure
** the r-tree table schema.
*/
- if( rc==SQLITE_OK ){
- if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){
- *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
- }else{
- char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]);
- char *zTmp;
- int ii;
- for(ii=4; zSql && iinBusy==1 );
- rtreeRelease(pRtree);
- }
+ pSql = sqlite3_str_new(db);
+ sqlite3_str_appendf(pSql, "CREATE TABLE x(%s", argv[3]);
+ for(ii=4; iinAux++;
+ sqlite3_str_appendf(pSql, ",%s", argv[ii]+1);
+ }else if( pRtree->nAux>0 ){
+ break;
+ }else{
+ pRtree->nDim2++;
+ sqlite3_str_appendf(pSql, ",%s", argv[ii]);
+ }
+ }
+ sqlite3_str_appendf(pSql, ");");
+ zSql = sqlite3_str_finish(pSql);
+ if( !zSql ){
+ rc = SQLITE_NOMEM;
+ }else if( iinDim = pRtree->nDim2/2;
+ if( pRtree->nDim<1 ){
+ iErr = 2;
+ }else if( pRtree->nDim2>RTREE_MAX_DIMENSIONS*2 ){
+ iErr = 3;
+ }else if( pRtree->nDim2 % 2 ){
+ iErr = 1;
+ }else{
+ iErr = 0;
+ }
+ if( iErr ){
+ *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]);
+ goto rtreeInit_fail;
+ }
+ pRtree->nBytesPerCell = 8 + pRtree->nDim2*4;
+
+ /* Figure out the node size to use. */
+ rc = getNodeSize(db, pRtree, isCreate, pzErr);
+ if( rc ) goto rtreeInit_fail;
+ rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate);
+ if( rc ){
+ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+ goto rtreeInit_fail;
+ }
+
+ *ppVtab = (sqlite3_vtab *)pRtree;
+ return SQLITE_OK;
+
+rtreeInit_fail:
+ if( rc==SQLITE_OK ) rc = SQLITE_ERROR;
+ assert( *ppVtab==0 );
+ assert( pRtree->nBusy==1 );
+ rtreeRelease(pRtree);
return rc;
}
/*
@@ -3606,10 +3775,482 @@
u8 *zBlob = (u8 *)sqlite3_value_blob(apArg[0]);
sqlite3_result_int(ctx, readInt16(zBlob));
}
}
+/*
+** Context object passed between the various routines that make up the
+** implementation of integrity-check function rtreecheck().
+*/
+typedef struct RtreeCheck RtreeCheck;
+struct RtreeCheck {
+ sqlite3 *db; /* Database handle */
+ const char *zDb; /* Database containing rtree table */
+ const char *zTab; /* Name of rtree table */
+ int bInt; /* True for rtree_i32 table */
+ int nDim; /* Number of dimensions for this rtree tbl */
+ sqlite3_stmt *pGetNode; /* Statement used to retrieve nodes */
+ sqlite3_stmt *aCheckMapping[2]; /* Statements to query %_parent/%_rowid */
+ int nLeaf; /* Number of leaf cells in table */
+ int nNonLeaf; /* Number of non-leaf cells in table */
+ int rc; /* Return code */
+ char *zReport; /* Message to report */
+ int nErr; /* Number of lines in zReport */
+};
+
+#define RTREE_CHECK_MAX_ERROR 100
+
+/*
+** Reset SQL statement pStmt. If the sqlite3_reset() call returns an error,
+** and RtreeCheck.rc==SQLITE_OK, set RtreeCheck.rc to the error code.
+*/
+static void rtreeCheckReset(RtreeCheck *pCheck, sqlite3_stmt *pStmt){
+ int rc = sqlite3_reset(pStmt);
+ if( pCheck->rc==SQLITE_OK ) pCheck->rc = rc;
+}
+
+/*
+** The second and subsequent arguments to this function are a format string
+** and printf style arguments. This function formats the string and attempts
+** to compile it as an SQL statement.
+**
+** If successful, a pointer to the new SQL statement is returned. Otherwise,
+** NULL is returned and an error code left in RtreeCheck.rc.
+*/
+static sqlite3_stmt *rtreeCheckPrepare(
+ RtreeCheck *pCheck, /* RtreeCheck object */
+ const char *zFmt, ... /* Format string and trailing args */
+){
+ va_list ap;
+ char *z;
+ sqlite3_stmt *pRet = 0;
+
+ va_start(ap, zFmt);
+ z = sqlite3_vmprintf(zFmt, ap);
+
+ if( pCheck->rc==SQLITE_OK ){
+ if( z==0 ){
+ pCheck->rc = SQLITE_NOMEM;
+ }else{
+ pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0);
+ }
+ }
+
+ sqlite3_free(z);
+ va_end(ap);
+ return pRet;
+}
+
+/*
+** The second and subsequent arguments to this function are a printf()
+** style format string and arguments. This function formats the string and
+** appends it to the report being accumuated in pCheck.
+*/
+static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){
+ va_list ap;
+ va_start(ap, zFmt);
+ if( pCheck->rc==SQLITE_OK && pCheck->nErrrc = SQLITE_NOMEM;
+ }else{
+ pCheck->zReport = sqlite3_mprintf("%z%s%z",
+ pCheck->zReport, (pCheck->zReport ? "\n" : ""), z
+ );
+ if( pCheck->zReport==0 ){
+ pCheck->rc = SQLITE_NOMEM;
+ }
+ }
+ pCheck->nErr++;
+ }
+ va_end(ap);
+}
+
+/*
+** This function is a no-op if there is already an error code stored
+** in the RtreeCheck object indicated by the first argument. NULL is
+** returned in this case.
+**
+** Otherwise, the contents of rtree table node iNode are loaded from
+** the database and copied into a buffer obtained from sqlite3_malloc().
+** If no error occurs, a pointer to the buffer is returned and (*pnNode)
+** is set to the size of the buffer in bytes.
+**
+** Or, if an error does occur, NULL is returned and an error code left
+** in the RtreeCheck object. The final value of *pnNode is undefined in
+** this case.
+*/
+static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){
+ u8 *pRet = 0; /* Return value */
+
+ assert( pCheck->rc==SQLITE_OK );
+ if( pCheck->pGetNode==0 ){
+ pCheck->pGetNode = rtreeCheckPrepare(pCheck,
+ "SELECT data FROM %Q.'%q_node' WHERE nodeno=?",
+ pCheck->zDb, pCheck->zTab
+ );
+ }
+
+ if( pCheck->rc==SQLITE_OK ){
+ sqlite3_bind_int64(pCheck->pGetNode, 1, iNode);
+ if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){
+ int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0);
+ const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0);
+ pRet = sqlite3_malloc(nNode);
+ if( pRet==0 ){
+ pCheck->rc = SQLITE_NOMEM;
+ }else{
+ memcpy(pRet, pNode, nNode);
+ *pnNode = nNode;
+ }
+ }
+ rtreeCheckReset(pCheck, pCheck->pGetNode);
+ if( pCheck->rc==SQLITE_OK && pRet==0 ){
+ rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode);
+ }
+ }
+
+ return pRet;
+}
+
+/*
+** This function is used to check that the %_parent (if bLeaf==0) or %_rowid
+** (if bLeaf==1) table contains a specified entry. The schemas of the
+** two tables are:
+**
+** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER)
+** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER, ...)
+**
+** In both cases, this function checks that there exists an entry with
+** IPK value iKey and the second column set to iVal.
+**
+*/
+static void rtreeCheckMapping(
+ RtreeCheck *pCheck, /* RtreeCheck object */
+ int bLeaf, /* True for a leaf cell, false for interior */
+ i64 iKey, /* Key for mapping */
+ i64 iVal /* Expected value for mapping */
+){
+ int rc;
+ sqlite3_stmt *pStmt;
+ const char *azSql[2] = {
+ "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?1",
+ "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?1"
+ };
+
+ assert( bLeaf==0 || bLeaf==1 );
+ if( pCheck->aCheckMapping[bLeaf]==0 ){
+ pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck,
+ azSql[bLeaf], pCheck->zDb, pCheck->zTab
+ );
+ }
+ if( pCheck->rc!=SQLITE_OK ) return;
+
+ pStmt = pCheck->aCheckMapping[bLeaf];
+ sqlite3_bind_int64(pStmt, 1, iKey);
+ rc = sqlite3_step(pStmt);
+ if( rc==SQLITE_DONE ){
+ rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table",
+ iKey, iVal, (bLeaf ? "%_rowid" : "%_parent")
+ );
+ }else if( rc==SQLITE_ROW ){
+ i64 ii = sqlite3_column_int64(pStmt, 0);
+ if( ii!=iVal ){
+ rtreeCheckAppendMsg(pCheck,
+ "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)",
+ iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal
+ );
+ }
+ }
+ rtreeCheckReset(pCheck, pStmt);
+}
+
+/*
+** Argument pCell points to an array of coordinates stored on an rtree page.
+** This function checks that the coordinates are internally consistent (no
+** x1>x2 conditions) and adds an error message to the RtreeCheck object
+** if they are not.
+**
+** Additionally, if pParent is not NULL, then it is assumed to point to
+** the array of coordinates on the parent page that bound the page
+** containing pCell. In this case it is also verified that the two
+** sets of coordinates are mutually consistent and an error message added
+** to the RtreeCheck object if they are not.
+*/
+static void rtreeCheckCellCoord(
+ RtreeCheck *pCheck,
+ i64 iNode, /* Node id to use in error messages */
+ int iCell, /* Cell number to use in error messages */
+ u8 *pCell, /* Pointer to cell coordinates */
+ u8 *pParent /* Pointer to parent coordinates */
+){
+ RtreeCoord c1, c2;
+ RtreeCoord p1, p2;
+ int i;
+
+ for(i=0; inDim; i++){
+ readCoord(&pCell[4*2*i], &c1);
+ readCoord(&pCell[4*(2*i + 1)], &c2);
+
+ /* printf("%e, %e\n", c1.u.f, c2.u.f); */
+ if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){
+ rtreeCheckAppendMsg(pCheck,
+ "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode
+ );
+ }
+
+ if( pParent ){
+ readCoord(&pParent[4*2*i], &p1);
+ readCoord(&pParent[4*(2*i + 1)], &p2);
+
+ if( (pCheck->bInt ? c1.ibInt ? c2.i>p2.i : c2.f>p2.f)
+ ){
+ rtreeCheckAppendMsg(pCheck,
+ "Dimension %d of cell %d on node %lld is corrupt relative to parent"
+ , i, iCell, iNode
+ );
+ }
+ }
+ }
+}
+
+/*
+** Run rtreecheck() checks on node iNode, which is at depth iDepth within
+** the r-tree structure. Argument aParent points to the array of coordinates
+** that bound node iNode on the parent node.
+**
+** If any problems are discovered, an error message is appended to the
+** report accumulated in the RtreeCheck object.
+*/
+static void rtreeCheckNode(
+ RtreeCheck *pCheck,
+ int iDepth, /* Depth of iNode (0==leaf) */
+ u8 *aParent, /* Buffer containing parent coords */
+ i64 iNode /* Node to check */
+){
+ u8 *aNode = 0;
+ int nNode = 0;
+
+ assert( iNode==1 || aParent!=0 );
+ assert( pCheck->nDim>0 );
+
+ aNode = rtreeCheckGetNode(pCheck, iNode, &nNode);
+ if( aNode ){
+ if( nNode<4 ){
+ rtreeCheckAppendMsg(pCheck,
+ "Node %lld is too small (%d bytes)", iNode, nNode
+ );
+ }else{
+ int nCell; /* Number of cells on page */
+ int i; /* Used to iterate through cells */
+ if( aParent==0 ){
+ iDepth = readInt16(aNode);
+ if( iDepth>RTREE_MAX_DEPTH ){
+ rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth);
+ sqlite3_free(aNode);
+ return;
+ }
+ }
+ nCell = readInt16(&aNode[2]);
+ if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){
+ rtreeCheckAppendMsg(pCheck,
+ "Node %lld is too small for cell count of %d (%d bytes)",
+ iNode, nCell, nNode
+ );
+ }else{
+ for(i=0; i